Fix [ERR_REQUIRE_ESM] error when importing hf_transformers (#5746)

* Fix [ERR_REQUIRE_ESM] error when importing hf_transformers * Use alternate fix, add export test --------- Co-authored-by: jacoblee93 <[email protected]>
langchain-ai · Jun 13, 2024 · 2fb4ac4 · 2fb4ac4
1 parent b1cef30
commit 2fb4ac4
Show file tree

Hide file tree

Showing 11 changed files with 23 additions and 22 deletions.
diff --git a/environment_tests/test-exports-cjs/package.json b/environment_tests/test-exports-cjs/package.json
@@ -26,6 +26,7 @@
     "@langchain/core": "workspace:*",
     "@langchain/openai": "workspace:*",
     "@tsconfig/recommended": "^1.0.2",
+    "@xenova/transformers": "^2.5.4",
     "d3-dsv": "2",
     "hnswlib-node": "^3.0.0",
     "langchain": "workspace:*",

diff --git a/environment_tests/test-exports-cjs/src/import.js b/environment_tests/test-exports-cjs/src/import.js
@@ -4,7 +4,7 @@ async function test() {
   const { LLMChain } = await import("langchain/chains");
   const { ChatPromptTemplate } = await import("@langchain/core/prompts");
   const { HNSWLib } = await import("@langchain/community/vectorstores/hnswlib");
-  const { OpenAIEmbeddings } = await import("@langchain/openai");
+  const { HuggingFaceTransformersEmbeddings } = await import("@langchain/community/embeddings/hf_transformers");
   const { Document } = await import("@langchain/core/documents");
   const { CSVLoader } = await import("langchain/document_loaders/fs/csv");
 
@@ -17,7 +17,7 @@ async function test() {
   // Test dynamic imports of peer dependencies
   const { HierarchicalNSW } = await HNSWLib.imports();
 
-  const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+  const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" }), {
     space: "ip",
     numDimensions: 3,
     index: new HierarchicalNSW("ip", 3),

diff --git a/environment_tests/test-exports-cjs/src/index.mjs b/environment_tests/test-exports-cjs/src/index.mjs
@@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai";
 import { LLMChain } from "langchain/chains";
 import { HNSWLib } from "@langchain/community/vectorstores/hnswlib";
 import { ChatPromptTemplate } from "@langchain/core/prompts";
-import { OpenAIEmbeddings } from "@langchain/openai";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
 import { Document } from "@langchain/core/documents";
 import { CSVLoader } from "langchain/document_loaders/fs/csv";
 
@@ -16,7 +16,7 @@ assert(typeof HNSWLib === "function");
 // Test dynamic imports of peer dependencies
 const { HierarchicalNSW } = await HNSWLib.imports();
 
-const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" }), {
   space: "ip",
   numDimensions: 3,
   index: new HierarchicalNSW("ip", 3),

diff --git a/environment_tests/test-exports-cjs/src/index.ts b/environment_tests/test-exports-cjs/src/index.ts
@@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai";
 import { LLMChain } from "langchain/chains";
 import { ChatPromptTemplate } from "@langchain/core/prompts";
 import { HNSWLib } from "@langchain/community/vectorstores/hnswlib";
-import { OpenAIEmbeddings } from "@langchain/openai";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
 import { Document } from "@langchain/core/documents";
 import { CSVLoader } from "langchain/document_loaders/fs/csv";
 
@@ -26,7 +26,7 @@ async function test(useAzure: boolean = false) {
     : {
         openAIApiKey: "sk-XXXX",
       };
-  const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+  const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" }), {
     space: "ip",
     numDimensions: 3,
     index: new HierarchicalNSW("ip", 3),

diff --git a/environment_tests/test-exports-cjs/src/require.js b/environment_tests/test-exports-cjs/src/require.js
@@ -3,7 +3,7 @@ const { OpenAI } = require("@langchain/openai");
 const { LLMChain } = require("langchain/chains");
 const { ChatPromptTemplate } = require("@langchain/core/prompts");
 const { HNSWLib } = require("@langchain/community/vectorstores/hnswlib");
-const { OpenAIEmbeddings } = require("@langchain/openai");
+const { HuggingFaceTransformersEmbeddings } = require("@langchain/community/embeddings/hf_transformers");
 const { Document } = require("@langchain/core/documents");
 const { CSVLoader } = require("langchain/document_loaders/fs/csv");
 
@@ -17,7 +17,7 @@ async function test() {
   // Test dynamic imports of peer dependencies
   const { HierarchicalNSW } = await HNSWLib.imports();
 
-  const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+  const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2" }), {
     space: "ip",
     numDimensions: 3,
     index: new HierarchicalNSW("ip", 3),

diff --git a/environment_tests/test-exports-esm/package.json b/environment_tests/test-exports-esm/package.json
@@ -27,6 +27,7 @@
     "@langchain/core": "workspace:*",
     "@langchain/openai": "workspace:*",
     "@tsconfig/recommended": "^1.0.2",
+    "@xenova/transformers": "^2.5.4",
     "d3-dsv": "2",
     "hnswlib-node": "^3.0.0",
     "langchain": "workspace:*",

diff --git a/environment_tests/test-exports-esm/src/import.cjs b/environment_tests/test-exports-esm/src/import.cjs
@@ -4,7 +4,7 @@ async function test() {
   const { LLMChain } = await import("langchain/chains");
   const { ChatPromptTemplate } = await import("@langchain/core/prompts");
   const { HNSWLib } = await import("@langchain/community/vectorstores/hnswlib");
-  const { OpenAIEmbeddings } = await import("@langchain/openai");
+  const { HuggingFaceTransformersEmbeddings } = await import("@langchain/community/embeddings/hf_transformers");
   const { Document } = await import("@langchain/core/documents");
   const { CSVLoader } = await import("langchain/document_loaders/fs/csv");
 
@@ -17,7 +17,7 @@ async function test() {
   // Test dynamic imports of peer dependencies
   const { HierarchicalNSW } = await HNSWLib.imports();
 
-  const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+  const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }), {
     space: "ip",
     numDimensions: 3,
     index: new HierarchicalNSW("ip", 3),

diff --git a/environment_tests/test-exports-esm/src/index.js b/environment_tests/test-exports-esm/src/index.js
@@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai";
 import { LLMChain } from "langchain/chains";
 import { ChatPromptTemplate } from "@langchain/core/prompts";
 import { HNSWLib } from "@langchain/community/vectorstores/hnswlib";
-import { OpenAIEmbeddings } from "@langchain/openai";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
 import { Document } from "@langchain/core/documents";
 import { CSVLoader } from "langchain/document_loaders/fs/csv";
 import { CallbackManager } from "@langchain/core/callbacks/manager";
@@ -13,13 +13,13 @@ assert(typeof OpenAI === "function");
 assert(typeof LLMChain === "function");
 assert(typeof ChatPromptTemplate === "function");
 assert(typeof HNSWLib === "function");
-assert(typeof OpenAIEmbeddings === "function");
+assert(typeof HuggingFaceTransformersEmbeddings === "function");
 assert(typeof CallbackManager === "function");
 
 // Test dynamic imports of peer dependencies
 const { HierarchicalNSW } = await HNSWLib.imports();
 
-const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }), {
   space: "ip",
   numDimensions: 3,
   index: new HierarchicalNSW("ip", 3),

diff --git a/environment_tests/test-exports-esm/src/index.ts b/environment_tests/test-exports-esm/src/index.ts
@@ -3,7 +3,7 @@ import { OpenAI } from "@langchain/openai";
 import { LLMChain } from "langchain/chains";
 import { ChatPromptTemplate } from "@langchain/core/prompts";
 import { HNSWLib } from "@langchain/community/vectorstores/hnswlib";
-import { OpenAIEmbeddings } from "@langchain/openai";
+import { HuggingFaceTransformersEmbeddings } from "@langchain/community/embeddings/hf_transformers";
 import { Document } from "@langchain/core/documents";
 import { CSVLoader } from "langchain/document_loaders/fs/csv";
 
@@ -27,7 +27,7 @@ async function test(useAzure: boolean = false) {
         openAIApiKey: "sk-XXXX",
       };
 
-  const vs = new HNSWLib(new OpenAIEmbeddings(openAIParameters), {
+  const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }), {
     space: "ip",
     numDimensions: 3,
     index: new HierarchicalNSW("ip", 3),

diff --git a/environment_tests/test-exports-esm/src/require.cjs b/environment_tests/test-exports-esm/src/require.cjs
@@ -3,7 +3,7 @@ const { OpenAI } = require("@langchain/openai");
 const { LLMChain } = require("langchain/chains");
 const { ChatPromptTemplate } = require("@langchain/core/prompts");
 const { HNSWLib } = require("@langchain/community/vectorstores/hnswlib");
-const { OpenAIEmbeddings } = require("@langchain/openai");
+const { HuggingFaceTransformersEmbeddings } = require("@langchain/community/embeddings/hf_transformers");
 const { Document } = require("@langchain/core/documents");
 const { CSVLoader } = require("langchain/document_loaders/fs/csv");
 
@@ -17,7 +17,7 @@ async function test() {
   // Test dynamic imports of peer dependencies
   const { HierarchicalNSW } = await HNSWLib.imports();
 
-  const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+  const vs = new HNSWLib(new HuggingFaceTransformersEmbeddings({ model: "Xenova/all-MiniLM-L6-v2", }), {
     space: "ip",
     numDimensions: 3,
     index: new HierarchicalNSW("ip", 3),

diff --git a/libs/langchain-community/src/embeddings/hf_transformers.ts b/libs/langchain-community/src/embeddings/hf_transformers.ts
@@ -1,4 +1,4 @@
-import { Pipeline, pipeline } from "@xenova/transformers";
+import type { Pipeline } from "@xenova/transformers";
 import { Embeddings, type EmbeddingsParams } from "@langchain/core/embeddings";
 import { chunkArray } from "@langchain/core/utils/chunk_array";
 
@@ -100,10 +100,9 @@ export class HuggingFaceTransformersEmbeddings
   }
 
   private async runEmbedding(texts: string[]) {
-    const pipe = await (this.pipelinePromise ??= pipeline(
-      "feature-extraction",
-      this.model
-    ));
+    const pipe = await (this.pipelinePromise ??= (
+      await import("@xenova/transformers")
+    ).pipeline("feature-extraction", this.model));
 
     return this.caller.call(async () => {
       const output = await pipe(texts, { pooling: "mean", normalize: true });