unstructured[major]: Unstructured partner package

libs/langchain-unstructured/.eslintrc.cjs

@@ -0,0 +1,74 @@
+module.exports = {
+  extends: [
+    "airbnb-base",
+    "eslint:recommended",
+    "prettier",
+    "plugin:@typescript-eslint/recommended",
+  ],
+  parserOptions: {
+    ecmaVersion: 12,
+    parser: "@typescript-eslint/parser",
+    project: "./tsconfig.json",
+    sourceType: "module",
+  },
+  plugins: ["@typescript-eslint", "no-instanceof"],
+  ignorePatterns: [
+    ".eslintrc.cjs",
+    "scripts",
+    "node_modules",
+    "dist",
+    "dist-cjs",
+    "*.js",
+    "*.cjs",
+    "*.d.ts",
+  ],
+  rules: {
+    "no-process-env": 2,
+    "no-instanceof/no-instanceof": 2,
+    "@typescript-eslint/explicit-module-boundary-types": 0,
+    "@typescript-eslint/no-empty-function": 0,
+    "@typescript-eslint/no-shadow": 0,
+    "@typescript-eslint/no-empty-interface": 0,
+    "@typescript-eslint/no-use-before-define": ["error", "nofunc"],
+    "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }],
+    "@typescript-eslint/no-floating-promises": "error",
+    "@typescript-eslint/no-misused-promises": "error",
+    camelcase: 0,
+    "class-methods-use-this": 0,
+    "import/extensions": [2, "ignorePackages"],
+    "import/no-extraneous-dependencies": [
+      "error",
+      { devDependencies: ["**/*.test.ts"] },
+    ],
+    "import/no-unresolved": 0,
+    "import/prefer-default-export": 0,
+    "keyword-spacing": "error",
+    "max-classes-per-file": 0,
+    "max-len": 0,
+    "no-await-in-loop": 0,
+    "no-bitwise": 0,
+    "no-console": 0,
+    "no-restricted-syntax": 0,
+    "no-shadow": 0,
+    "no-continue": 0,
+    "no-void": 0,
+    "no-underscore-dangle": 0,
+    "no-use-before-define": 0,
+    "no-useless-constructor": 0,
+    "no-return-await": 0,
+    "consistent-return": 0,
+    "no-else-return": 0,
+    "func-names": 0,
+    "no-lonely-if": 0,
+    "prefer-rest-params": 0,
+    "new-cap": ["error", { properties: false, capIsNew: false }],
+  },
+  overrides: [
+    {
+      files: ["**/*.test.ts"],
+      rules: {
+        "@typescript-eslint/no-unused-vars": "off",
+      },
+    },
+  ],
+};

libs/langchain-unstructured/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+dist
+.yarn

libs/langchain-unstructured/.prettierrc

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://json.schemastore.org/prettierrc",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "useTabs": false,
+  "semi": true,
+  "singleQuote": false,
+  "quoteProps": "as-needed",
+  "jsxSingleQuote": false,
+  "trailingComma": "es5",
+  "bracketSpacing": true,
+  "arrowParens": "always",
+  "requirePragma": false,
+  "insertPragma": false,
+  "proseWrap": "preserve",
+  "htmlWhitespaceSensitivity": "css",
+  "vueIndentScriptAndStyle": false,
+  "endOfLine": "lf"
+}

libs/langchain-unstructured/.release-it.json

@@ -0,0 +1,10 @@
+{
+  "github": {
+    "release": true,
+    "autoGenerate": true,
+    "tokenRef": "GITHUB_TOKEN_RELEASE"
+  },
+  "npm": {
+    "versionArgs": ["--workspaces-update=false"]
+  }
+}

libs/langchain-unstructured/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2023 LangChain
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

libs/langchain-unstructured/jest.config.cjs

@@ -0,0 +1,21 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: "ts-jest/presets/default-esm",
+  testEnvironment: "./jest.env.cjs",
+  modulePathIgnorePatterns: ["dist/", "docs/"],
+  moduleNameMapper: {
+    "^(\\.{1,2}/.*)\\.js$": "$1",
+  },
+  transform: {
+    "^.+\\.tsx?$": ["@swc/jest"],
+  },
+  transformIgnorePatterns: [
+    "/node_modules/",
+    "\\.pnp\\.[^\\/]+$",
+    "./scripts/jest-setup-after-env.js",
+  ],
+  setupFiles: ["dotenv/config"],
+  testTimeout: 20_000,
+  passWithNoTests: true,
+  collectCoverageFrom: ["src/**/*.ts"],
+};

libs/langchain-unstructured/jest.env.cjs

@@ -0,0 +1,12 @@
+const { TestEnvironment } = require("jest-environment-node");
+
+class AdjustedTestEnvironmentToSupportFloat32Array extends TestEnvironment {
+  constructor(config, context) {
+    // Make `instanceof Float32Array` return true in tests
+    // to avoid https://github.com/xenova/transformers.js/issues/57 and https://github.com/jestjs/jest/issues/2549
+    super(config, context);
+    this.global.Float32Array = Float32Array;
+  }
+}
+
+module.exports = AdjustedTestEnvironmentToSupportFloat32Array;

libs/langchain-unstructured/langchain.config.js

@@ -0,0 +1,22 @@
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/**
+ * @param {string} relativePath
+ * @returns {string}
+ */
+function abs(relativePath) {
+  return resolve(dirname(fileURLToPath(import.meta.url)), relativePath);
+}
+
+export const config = {
+  internals: [/node\:/, /@langchain\/core\//],
+  entrypoints: {
+    index: "index",
+  },
+  requiresOptionalDependency: [],
+  tsConfigPath: resolve("./tsconfig.json"),
+  cjsSource: "./dist-cjs",
+  cjsDestination: "./dist",
+  abs,
+};

libs/langchain-unstructured/package.json

@@ -0,0 +1,79 @@
+{
+  "name": "@langchain/unstructured",
+  "version": "0.0.0",
+  "description": "Sample integration for LangChain.js",
+  "type": "module",
+  "engines": {
+    "node": ">=18"
+  },
+  "main": "./index.js",
+  "types": "./index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "[email protected]:langchain-ai/langchainjs.git"
+  },
+  "homepage": "https://github.com/langchain-ai/langchainjs/tree/main/libs/langchain-unstructured/",
+  "scripts": {
+    "build": "yarn turbo:command build:internal --filter=@langchain/unstructured",
+    "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking",
+    "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/",
+    "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts",
+    "lint": "yarn lint:eslint && yarn lint:dpdm",
+    "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm",
+    "clean": "rm -rf .turbo dist/",
+    "prepack": "yarn build",
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%",
+    "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts",
+    "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000",
+    "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%",
+    "format": "prettier --config .prettierrc --write \"src\"",
+    "format:check": "prettier --config .prettierrc --check \"src\""
+  },
+  "author": "LangChain",
+  "license": "MIT",
+  "dependencies": {
+    "@langchain/core": ">0.1.0 <0.3.0",
+    "unstructured-client": "^0.13.0"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.5.0",
+    "@langchain/scripts": "~0.0.14",
+    "@swc/core": "^1.3.90",
+    "@swc/jest": "^0.2.29",
+    "@tsconfig/recommended": "^1.0.3",
+    "@typescript-eslint/eslint-plugin": "^6.12.0",
+    "@typescript-eslint/parser": "^6.12.0",
+    "dotenv": "^16.3.1",
+    "dpdm": "^3.12.0",
+    "eslint": "^8.33.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-config-prettier": "^8.6.0",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-no-instanceof": "^1.0.1",
+    "eslint-plugin-prettier": "^4.2.1",
+    "jest": "^29.5.0",
+    "jest-environment-node": "^29.6.4",
+    "prettier": "^2.8.3",
+    "release-it": "^15.10.1",
+    "rollup": "^4.5.2",
+    "ts-jest": "^29.1.0",
+    "typescript": "<5.2.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "import": "./index.js",
+      "require": "./index.cjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist/",
+    "index.cjs",
+    "index.js",
+    "index.d.ts"
+  ]
+}

libs/langchain-unstructured/scripts/jest-setup-after-env.js

@@ -0,0 +1,9 @@
+import { awaitAllCallbacks } from "@langchain/core/callbacks/promises";
+import { afterAll, jest } from "@jest/globals";
+
+afterAll(awaitAllCallbacks);
+
+// Allow console.log to be disabled in tests
+if (process.env.DISABLE_CONSOLE_LOGS === "true") {
+  console.log = jest.fn();
+}