From e33ecf4fafdd6e478a1be9e4d3f44f330576867e Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Thu, 13 Jun 2024 13:56:36 -0700 Subject: [PATCH] Use alternate fix, add export test --- environment_tests/test-exports-cjs/package.json | 1 + environment_tests/test-exports-cjs/src/import.js | 4 ++-- environment_tests/test-exports-cjs/src/index.mjs | 4 ++-- environment_tests/test-exports-cjs/src/index.ts | 4 ++-- environment_tests/test-exports-cjs/src/require.js | 4 ++-- environment_tests/test-exports-esm/package.json | 1 + environment_tests/test-exports-esm/src/import.cjs | 4 ++-- environment_tests/test-exports-esm/src/index.js | 6 +++--- environment_tests/test-exports-esm/src/index.ts | 4 ++-- environment_tests/test-exports-esm/src/require.cjs | 4 ++-- .../src/embeddings/hf_transformers.ts | 12 ++++-------- 11 files changed, 23 insertions(+), 25 deletions(-) diff --git a/environment_tests/test-exports-cjs/package.json b/environment_tests/test-exports-cjs/package.json index c9f23b948d79..c97c2931d5cb 100644 --- a/environment_tests/test-exports-cjs/package.json +++ b/environment_tests/test-exports-cjs/package.json @@ -26,6 +26,7 @@ "@langchain/core": "workspace:*", "@langchain/openai": "workspace:*", "@tsconfig/recommended": "^1.0.2", + "@xenova/transformers": "^2.5.4", "d3-dsv": "2", "hnswlib-node": "^3.0.0", "langchain": "workspace:*", diff --git a/environment_tests/test-exports-cjs/src/import.js b/environment_tests/test-exports-cjs/src/import.js index 0de8ec153bcf..18a6fd161c3a 100644 --- a/environment_tests/test-exports-cjs/src/import.js +++ b/environment_tests/test-exports-cjs/src/import.js @@ -4,7 +4,7 @@ async function test() { const { LLMChain } = await import("langchain/chains"); const { ChatPromptTemplate } = await import("@langchain/core/prompts"); const { HNSWLib } = await import("@langchain/community/vectorstores/hnswlib"); - const { OpenAIEmbeddings } = await import("@langchain/openai"); + const { HuggingFaceTransformersEmbeddings } = await import("@langchain/community/embeddings/hf_transformers"); const { Document } = await import("@langchain/core/documents"); const { CSVLoader } = await import("langchain/document_loaders/fs/csv"); @@ -17,7 +17,7 @@ async function test() { // Test dynamic imports of peer dependencies const { HierarchicalNSW } = await HNSWLib.imports(); - const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), { + const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" }), { space: "ip", numDimensions: 3, index: new HierarchicalNSW("ip", 3), diff --git a/environment_tests/test-exports-cjs/src/index.mjs b/environment_tests/test-exports-cjs/src/index.mjs index 4c957f9f8c91..90028e732c3d 100644 --- a/environment_tests/test-exports-cjs/src/index.mjs +++ b/environment_tests/test-exports-cjs/src/index.mjs @@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai"; import { LLMChain } from "langchain/chains"; import { HNSWLib } from "@langchain/community/vectorstores/hnswlib"; import { ChatPromptTemplate } from "@langchain/core/prompts"; -import { OpenAIEmbeddings } from "@langchain/openai"; +import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers"; import { Document } from "@langchain/core/documents"; import { CSVLoader } from "langchain/document_loaders/fs/csv"; @@ -16,7 +16,7 @@ assert(typeof HNSWLib === "function"); // Test dynamic imports of peer dependencies const { HierarchicalNSW } = await HNSWLib.imports(); -const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), { +const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" }), { space: "ip", numDimensions: 3, index: new HierarchicalNSW("ip", 3), diff --git a/environment_tests/test-exports-cjs/src/index.ts b/environment_tests/test-exports-cjs/src/index.ts index 1def3148b013..f28f606a28e0 100644 --- a/environment_tests/test-exports-cjs/src/index.ts +++ b/environment_tests/test-exports-cjs/src/index.ts @@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai"; import { LLMChain } from "langchain/chains"; import { ChatPromptTemplate } from "@langchain/core/prompts"; import { HNSWLib } from "@langchain/community/vectorstores/hnswlib"; -import { OpenAIEmbeddings } from "@langchain/openai"; +import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers"; import { Document } from "@langchain/core/documents"; import { CSVLoader } from "langchain/document_loaders/fs/csv"; @@ -26,7 +26,7 @@ async function test(useAzure: boolean = false) { : { openAIApiKey: "sk-XXXX", }; - const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), { + const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" }), { space: "ip", numDimensions: 3, index: new HierarchicalNSW("ip", 3), diff --git a/environment_tests/test-exports-cjs/src/require.js b/environment_tests/test-exports-cjs/src/require.js index 8e2acb525ce8..a9d0f32a12fc 100644 --- a/environment_tests/test-exports-cjs/src/require.js +++ b/environment_tests/test-exports-cjs/src/require.js @@ -3,7 +3,7 @@ const { OpenAI } = require("@langchain/openai"); const { LLMChain } = require("langchain/chains"); const { ChatPromptTemplate } = require("@langchain/core/prompts"); const { HNSWLib } = require("@langchain/community/vectorstores/hnswlib"); -const { OpenAIEmbeddings } = require("@langchain/openai"); +const { HuggingFaceTransformersEmbeddings } = require("@langchain/community/embeddings/hf_transformers"); const { Document } = require("@langchain/core/documents"); const { CSVLoader } = require("langchain/document_loaders/fs/csv"); @@ -17,7 +17,7 @@ async function test() { // Test dynamic imports of peer dependencies const { HierarchicalNSW } = await HNSWLib.imports(); - const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), { + const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" }), { space: "ip", numDimensions: 3, index: new HierarchicalNSW("ip", 3), diff --git a/environment_tests/test-exports-esm/package.json b/environment_tests/test-exports-esm/package.json index b3971f74bd14..ec04572c39cf 100644 --- a/environment_tests/test-exports-esm/package.json +++ b/environment_tests/test-exports-esm/package.json @@ -27,6 +27,7 @@ "@langchain/core": "workspace:*", "@langchain/openai": "workspace:*", "@tsconfig/recommended": "^1.0.2", + "@xenova/transformers": "^2.5.4", "d3-dsv": "2", "hnswlib-node": "^3.0.0", "langchain": "workspace:*", diff --git a/environment_tests/test-exports-esm/src/import.cjs b/environment_tests/test-exports-esm/src/import.cjs index 0de8ec153bcf..7641af80677b 100644 --- a/environment_tests/test-exports-esm/src/import.cjs +++ b/environment_tests/test-exports-esm/src/import.cjs @@ -4,7 +4,7 @@ async function test() { const { LLMChain } = await import("langchain/chains"); const { ChatPromptTemplate } = await import("@langchain/core/prompts"); const { HNSWLib } = await import("@langchain/community/vectorstores/hnswlib"); - const { OpenAIEmbeddings } = await import("@langchain/openai"); + const { HuggingFaceTransformersEmbeddings } = await import("@langchain/community/embeddings/hf_transformers"); const { Document } = await import("@langchain/core/documents"); const { CSVLoader } = await import("langchain/document_loaders/fs/csv"); @@ -17,7 +17,7 @@ async function test() { // Test dynamic imports of peer dependencies const { HierarchicalNSW } = await HNSWLib.imports(); - const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), { + const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }), { space: "ip", numDimensions: 3, index: new HierarchicalNSW("ip", 3), diff --git a/environment_tests/test-exports-esm/src/index.js b/environment_tests/test-exports-esm/src/index.js index 81f7dd48e4e7..952081a6c5af 100644 --- a/environment_tests/test-exports-esm/src/index.js +++ b/environment_tests/test-exports-esm/src/index.js @@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai"; import { LLMChain } from "langchain/chains"; import { ChatPromptTemplate } from "@langchain/core/prompts"; import { HNSWLib } from "@langchain/community/vectorstores/hnswlib"; -import { OpenAIEmbeddings } from "@langchain/openai"; +import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers"; import { Document } from "@langchain/core/documents"; import { CSVLoader } from "langchain/document_loaders/fs/csv"; import { CallbackManager } from "@langchain/core/callbacks/manager"; @@ -13,13 +13,13 @@ assert(typeof OpenAI === "function"); assert(typeof LLMChain === "function"); assert(typeof ChatPromptTemplate === "function"); assert(typeof HNSWLib === "function"); -assert(typeof OpenAIEmbeddings === "function"); +assert(typeof HuggingFaceTransformersEmbeddings === "function"); assert(typeof CallbackManager === "function"); // Test dynamic imports of peer dependencies const { HierarchicalNSW } = await HNSWLib.imports(); -const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), { +const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }), { space: "ip", numDimensions: 3, index: new HierarchicalNSW("ip", 3), diff --git a/environment_tests/test-exports-esm/src/index.ts b/environment_tests/test-exports-esm/src/index.ts index 7e611b34ba88..8dcd5f46aeb1 100644 --- a/environment_tests/test-exports-esm/src/index.ts +++ b/environment_tests/test-exports-esm/src/index.ts @@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai"; import { LLMChain } from "langchain/chains"; import { ChatPromptTemplate } from "@langchain/core/prompts"; import { HNSWLib } from "@langchain/community/vectorstores/hnswlib"; -import { OpenAIEmbeddings } from "@langchain/openai"; +import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers"; import { Document } from "@langchain/core/documents"; import { CSVLoader } from "langchain/document_loaders/fs/csv"; @@ -27,7 +27,7 @@ async function test(useAzure: boolean = false) { openAIApiKey: "sk-XXXX", }; - const vs = new HNSWLib(new OpenAIEmbeddings(openAIParameters), { + const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }), { space: "ip", numDimensions: 3, index: new HierarchicalNSW("ip", 3), diff --git a/environment_tests/test-exports-esm/src/require.cjs b/environment_tests/test-exports-esm/src/require.cjs index 8e2acb525ce8..4652414d937b 100644 --- a/environment_tests/test-exports-esm/src/require.cjs +++ b/environment_tests/test-exports-esm/src/require.cjs @@ -3,7 +3,7 @@ const { OpenAI } = require("@langchain/openai"); const { LLMChain } = require("langchain/chains"); const { ChatPromptTemplate } = require("@langchain/core/prompts"); const { HNSWLib } = require("@langchain/community/vectorstores/hnswlib"); -const { OpenAIEmbeddings } = require("@langchain/openai"); +const { HuggingFaceTransformersEmbeddings } = require("@langchain/community/embeddings/hf_transformers"); const { Document } = require("@langchain/core/documents"); const { CSVLoader } = require("langchain/document_loaders/fs/csv"); @@ -17,7 +17,7 @@ async function test() { // Test dynamic imports of peer dependencies const { HierarchicalNSW } = await HNSWLib.imports(); - const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), { + const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }), { space: "ip", numDimensions: 3, index: new HierarchicalNSW("ip", 3), diff --git a/libs/langchain-community/src/embeddings/hf_transformers.ts b/libs/langchain-community/src/embeddings/hf_transformers.ts index adcde32fe7fe..a196e8f989e6 100644 --- a/libs/langchain-community/src/embeddings/hf_transformers.ts +++ b/libs/langchain-community/src/embeddings/hf_transformers.ts @@ -1,8 +1,6 @@ -import { Pipeline } from "@xenova/transformers"; +import type { Pipeline } from "@xenova/transformers"; import { Embeddings, type EmbeddingsParams } from "@langchain/core/embeddings"; import { chunkArray } from "@langchain/core/utils/chunk_array"; -const TransformersApi = Function('return import("@xenova/transformers")')(); - export interface HuggingFaceTransformersEmbeddingsParams extends EmbeddingsParams { @@ -102,11 +100,9 @@ export class HuggingFaceTransformersEmbeddings } private async runEmbedding(texts: string[]) { - const { pipeline } = await TransformersApi; - const pipe = await (this.pipelinePromise ??= pipeline( - "feature-extraction", - this.model - )); + const pipe = await (this.pipelinePromise ??= ( + await import("@xenova/transformers") + ).pipeline("feature-extraction", this.model)); return this.caller.call(async () => { const output = await pipe(texts, { pooling: "mean", normalize: true });