Add test-exports-esbuild (langchain-ai#1170)

* Add test-exports-esbuild

* Test in CI

* Add additional external for docker build

docker-compose.yml

@@ -1,5 +1,13 @@
 version: '3'
 services:
+  test-exports-esbuild:
+    image: node:18
+    working_dir: /app
+    volumes:
+      - ./test-exports-esbuild:/package
+      - ./langchain:/langchain
+      - ./scripts:/scripts
+    command: bash /scripts/docker-ci-entrypoint.sh
   test-exports-esm:
     image: node:18
     working_dir: /app
@@ -52,6 +60,8 @@ services:
     image: alpine:3.14
     command: echo "Success"
     depends_on:
+      test-exports-esbuild:
+        condition: service_completed_successfully
       test-exports-esm:
         condition: service_completed_successfully
       test-exports-cjs:

package.json

@@ -9,6 +9,7 @@
     "langchain",
     "examples",
     "docs",
+    "test-exports-esbuild",
     "test-exports-esm",
     "test-exports-cjs",
     "test-exports-cf",

test-exports-esbuild/.gitignore

@@ -0,0 +1,2 @@
+dist-esm
+dist-cjs

test-exports-esbuild/entrypoint.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+DIR=$1
+
+for file in $DIR/*; do
+  if [[ $file == 'dist-esm/require.js' ]]; then
+    continue;
+  fi
+  node $file;
+done

test-exports-esbuild/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "test-exports-esbuild",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Tests for the things exported by the langchain package",
+  "main": "./index.mjs",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "test": "yarn run test:esm && yarn run test:cjs",
+    "test:esm": "rm -rf dist-esm && esbuild --bundle src/* --outdir=dist-esm --platform=node --format=esm --external:'../node_modules/*' --external:'./node_modules/*' --supported:top-level-await=true && bash ./entrypoint.sh dist-esm",
+    "test:cjs": "rm -rf dist-cjs && esbuild --bundle src/*.cjs --outdir=dist-cjs --platform=node --format=cjs --external:'../node_modules/*' --external:'./node_modules/*' --out-extension:.js=.cjs && bash ./entrypoint.sh dist-cjs",
+    "format": "prettier --write src",
+    "format:check": "prettier --check src"
+  },
+  "author": "LangChain",
+  "license": "MIT",
+  "dependencies": {
+    "d3-dsv": "2",
+    "hnswlib-node": "^1.4.2",
+    "langchain": "workspace:*"
+  },
+  "devDependencies": {
+    "@tsconfig/recommended": "^1.0.2",
+    "@types/node": "^18.15.11",
+    "esbuild": "^0.17.18",
+    "prettier": "^2.8.3",
+    "typescript": "^5.0.0"
+  }
+}

test-exports-esbuild/src/entrypoints.js

@@ -0,0 +1,36 @@
+import * as agents from "langchain/agents";
+import * as base_language from "langchain/base_language";
+import * as tools from "langchain/tools";
+import * as chains from "langchain/chains";
+import * as embeddings_base from "langchain/embeddings/base";
+import * as embeddings_fake from "langchain/embeddings/fake";
+import * as embeddings_openai from "langchain/embeddings/openai";
+import * as llms_base from "langchain/llms/base";
+import * as llms_openai from "langchain/llms/openai";
+import * as prompts from "langchain/prompts";
+import * as vectorstores_base from "langchain/vectorstores/base";
+import * as vectorstores_memory from "langchain/vectorstores/memory";
+import * as vectorstores_prisma from "langchain/vectorstores/prisma";
+import * as text_splitter from "langchain/text_splitter";
+import * as memory from "langchain/memory";
+import * as document from "langchain/document";
+import * as docstore from "langchain/docstore";
+import * as document_loaders_base from "langchain/document_loaders/base";
+import * as chat_models_base from "langchain/chat_models/base";
+import * as chat_models_openai from "langchain/chat_models/openai";
+import * as chat_models_anthropic from "langchain/chat_models/anthropic";
+import * as schema from "langchain/schema";
+import * as schema_output_parser from "langchain/schema/output_parser";
+import * as callbacks from "langchain/callbacks";
+import * as output_parsers from "langchain/output_parsers";
+import * as retrievers_remote from "langchain/retrievers/remote";
+import * as retrievers_databerry from "langchain/retrievers/databerry";
+import * as retrievers_contextual_compression from "langchain/retrievers/contextual_compression";
+import * as retrievers_document_compressors from "langchain/retrievers/document_compressors";
+import * as retrievers_time_weighted from "langchain/retrievers/time_weighted";
+import * as retrievers_document_compressors_chain_extract from "langchain/retrievers/document_compressors/chain_extract";
+import * as retrievers_hyde from "langchain/retrievers/hyde";
+import * as cache from "langchain/cache";
+import * as stores_file_in_memory from "langchain/stores/file/in_memory";
+import * as experimental_autogpt from "langchain/experimental/autogpt";
+import * as experimental_babyagi from "langchain/experimental/babyagi";

test-exports-esbuild/src/import.cjs

@@ -0,0 +1,59 @@
+async function test() {
+  const { default: assert } = await import("assert");
+  const { OpenAI } = await import("langchain/llms/openai");
+  const { LLMChain } = await import("langchain/chains");
+  const { ChatPromptTemplate } = await import("langchain/prompts");
+  const { loadPrompt } = await import("langchain/prompts/load");
+  const { HNSWLib } = await import("langchain/vectorstores/hnswlib");
+  const { OpenAIEmbeddings } = await import("langchain/embeddings/openai");
+  const { InMemoryDocstore, Document } = await import("langchain/docstore");
+  const { CSVLoader } = await import("langchain/document_loaders/fs/csv");
+
+  // Test exports
+  assert(typeof OpenAI === "function");
+  assert(typeof LLMChain === "function");
+  assert(typeof loadPrompt === "function");
+  assert(typeof ChatPromptTemplate === "function");
+  assert(typeof HNSWLib === "function");
+
+  // Test dynamic imports of peer dependencies
+  const { HierarchicalNSW } = await HNSWLib.imports();
+
+  const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+    space: "ip",
+    numDimensions: 3,
+    docstore: new InMemoryDocstore(),
+    index: new HierarchicalNSW("ip", 3),
+  });
+
+  await vs.addVectors(
+    [
+      [0, 1, 0],
+      [0, 0, 1],
+    ],
+    [
+      new Document({
+        pageContent: "a",
+      }),
+      new Document({
+        pageContent: "b",
+      }),
+    ]
+  );
+
+  assert((await vs.similaritySearchVectorWithScore([0, 0, 1], 1)).length === 1);
+
+  // Test CSVLoader
+  const loader = new CSVLoader(new Blob(["a,b,c\n1,2,3\n4,5,6"]));
+
+  const docs = await loader.load();
+
+  assert(docs.length === 2);
+}
+
+test()
+  .then(() => console.log("success"))
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });

test-exports-esbuild/src/index.js

@@ -0,0 +1,53 @@
+import assert from "assert";
+import { OpenAI } from "langchain/llms/openai";
+import { LLMChain } from "langchain/chains";
+import { ChatPromptTemplate } from "langchain/prompts";
+import { loadPrompt } from "langchain/prompts/load";
+import { HNSWLib } from "langchain/vectorstores/hnswlib";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+import { InMemoryDocstore, Document } from "langchain/docstore";
+import { CSVLoader } from "langchain/document_loaders/fs/csv";
+import { CallbackManager } from "langchain/callbacks";
+
+// Test exports
+assert(typeof OpenAI === "function");
+assert(typeof LLMChain === "function");
+assert(typeof loadPrompt === "function");
+assert(typeof ChatPromptTemplate === "function");
+assert(typeof HNSWLib === "function");
+assert(typeof OpenAIEmbeddings === "function");
+assert(typeof CallbackManager === "function");
+
+// Test dynamic imports of peer dependencies
+const { HierarchicalNSW } = await HNSWLib.imports();
+
+const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+  space: "ip",
+  numDimensions: 3,
+  docstore: new InMemoryDocstore(),
+  index: new HierarchicalNSW("ip", 3),
+});
+
+await vs.addVectors(
+  [
+    [0, 1, 0],
+    [0, 0, 1],
+  ],
+  [
+    new Document({
+      pageContent: "a",
+    }),
+    new Document({
+      pageContent: "b",
+    }),
+  ]
+);
+
+assert((await vs.similaritySearchVectorWithScore([0, 0, 1], 1)).length === 1);
+
+// Test CSVLoader
+const loader = new CSVLoader(new Blob(["a,b,c\n1,2,3\n4,5,6"]));
+
+const docs = await loader.load();
+
+assert(docs.length === 2);

test-exports-esbuild/src/require.cjs

@@ -0,0 +1,59 @@
+const assert = require("assert");
+const { OpenAI } = require("langchain/llms/openai");
+const { LLMChain } = require("langchain/chains");
+const { ChatPromptTemplate } = require("langchain/prompts");
+const { loadPrompt } = require("langchain/prompts/load");
+const { HNSWLib } = require("langchain/vectorstores/hnswlib");
+const { OpenAIEmbeddings } = require("langchain/embeddings/openai");
+const { InMemoryDocstore, Document } = require("langchain/docstore");
+const { CSVLoader } = require("langchain/document_loaders/fs/csv");
+
+async function test() {
+  // Test exports
+  assert(typeof OpenAI === "function");
+  assert(typeof LLMChain === "function");
+  assert(typeof loadPrompt === "function");
+  assert(typeof ChatPromptTemplate === "function");
+  assert(typeof HNSWLib === "function");
+
+  // Test dynamic imports of peer dependencies
+  const { HierarchicalNSW } = await HNSWLib.imports();
+
+  const vs = new HNSWLib(new OpenAIEmbeddings({ openAIApiKey: "sk-XXXX" }), {
+    space: "ip",
+    numDimensions: 3,
+    docstore: new InMemoryDocstore(),
+    index: new HierarchicalNSW("ip", 3),
+  });
+
+  await vs.addVectors(
+    [
+      [0, 1, 0],
+      [0, 0, 1],
+    ],
+    [
+      new Document({
+        pageContent: "a",
+      }),
+      new Document({
+        pageContent: "b",
+      }),
+    ]
+  );
+
+  assert((await vs.similaritySearchVectorWithScore([0, 0, 1], 1)).length === 1);
+
+  // Test CSVLoader
+  const loader = new CSVLoader(new Blob(["a,b,c\n1,2,3\n4,5,6"]));
+
+  const docs = await loader.load();
+
+  assert(docs.length === 2);
+}
+
+test()
+  .then(() => console.log("success"))
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });

test-exports-esbuild/src/typescript.ts

@@ -0,0 +1,75 @@
+import assert from "assert";
+import { OpenAI } from "langchain/llms/openai";
+import { LLMChain } from "langchain/chains";
+import { ChatPromptTemplate } from "langchain/prompts";
+import { loadPrompt } from "langchain/prompts/load";
+import { HNSWLib } from "langchain/vectorstores/hnswlib";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+import { InMemoryDocstore, Document } from "langchain/docstore";
+import { CSVLoader } from "langchain/document_loaders/fs/csv";
+
+async function test(useAzure: boolean = false) {
+  // Test exports
+  assert(typeof OpenAI === "function");
+  assert(typeof LLMChain === "function");
+  assert(typeof loadPrompt === "function");
+  assert(typeof ChatPromptTemplate === "function");
+  assert(typeof HNSWLib === "function");
+
+  // Test dynamic imports of peer dependencies
+  const { HierarchicalNSW } = await HNSWLib.imports();
+  const openAIParameters = useAzure
+    ? {
+        azureOpenAIApiKey: "sk-XXXX",
+        azureOpenAIApiInstanceName: "XXXX",
+        azureOpenAIApiDeploymentName: "XXXX",
+        azureOpenAIApiVersion: "XXXX",
+      }
+    : {
+        openAIApiKey: "sk-XXXX",
+      };
+
+  const vs = new HNSWLib(new OpenAIEmbeddings(openAIParameters), {
+    space: "ip",
+    numDimensions: 3,
+    docstore: new InMemoryDocstore(),
+    index: new HierarchicalNSW("ip", 3),
+  });
+
+  await vs.addVectors(
+    [
+      [0, 1, 0],
+      [0, 0, 1],
+    ],
+    [
+      new Document({
+        pageContent: "a",
+      }),
+      new Document({
+        pageContent: "b",
+      }),
+    ]
+  );
+
+  assert((await vs.similaritySearchVectorWithScore([0, 0, 1], 1)).length === 1);
+
+  // Test CSVLoader
+  const loader = new CSVLoader(new Blob(["a,b,c\n1,2,3\n4,5,6"]));
+
+  const docs = await loader.load();
+
+  assert(docs.length === 2);
+}
+
+test(false)
+  .then(() => console.log("openAI Api success"))
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });
+test(true)
+  .then(() => console.log("Azure openAI Api success"))
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });

test-exports-esbuild/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "@tsconfig/recommended",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "lib": [
+      "ES2021",
+      "ES2022.Object",
+      "DOM"
+    ],
+    "target": "ES2021",
+    "module": "nodenext",
+  },
+  "include": [
+    "src/**/*"
+  ]
+}