Merge pull request #154 from llm-tools/loaders

Added xml loader
llm-tools · Nov 4, 2024 · 73cd1e7 · 73cd1e7
2 parents 959e184 + 3bf6b07
commit 73cd1e7
Show file tree

Hide file tree

Showing 78 changed files with 1,301 additions and 710 deletions.
diff --git a/core/embedjs-interfaces/CHANGELOG.md b/core/embedjs-interfaces/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
-        "@langchain/core": "^0.3.16",
+        "@langchain/core": "^0.3.17",
         "debug": "^4.3.7",
         "md5": "^2.3.0",
         "uuid": "^11.0.2"

diff --git a/core/embedjs-utils/CHANGELOG.md b/core/embedjs-utils/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.15"
+        "@llm-tools/embedjs-interfaces": "0.1.16"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs-utils/src/index.ts b/core/embedjs-utils/src/index.ts
@@ -2,3 +2,4 @@ export * from './util/arrays.js';
 export * from './util/log.js';
 export * from './util/stream.js';
 export * from './util/strings.js';
+export * from './util/web.js';
diff --git a/core/embedjs-utils/src/util/web.ts b/core/embedjs-utils/src/util/web.ts
@@ -0,0 +1,45 @@
+import createDebugMessages from 'debug';
+
+const DEFAULT_USER_AGENT =
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36';
+
+type getSafeResponsePartial = {
+    headers: Headers;
+    statusCode: number;
+};
+
+export async function getSafe(
+    url: string,
+    options: { headers?: Record<string, string>; format: 'text' },
+): Promise<{ body: string } & getSafeResponsePartial>;
+export async function getSafe(
+    url: string,
+    options: { headers?: Record<string, string>; format: 'buffer' },
+): Promise<{ body: Buffer } & getSafeResponsePartial>;
+export async function getSafe(
+    url: string,
+    options?: { headers?: Record<string, string>; format?: 'stream' },
+): Promise<{ body: NodeJS.ReadableStream } & getSafeResponsePartial>;
+export async function getSafe(
+    url: string,
+    options?: { headers?: Record<string, string>; format?: 'text' | 'stream' | 'buffer' },
+) {
+    const headers = options?.headers ?? {};
+    headers['User-Agent'] = headers['User-Agent'] ?? DEFAULT_USER_AGENT;
+
+    const format = options?.format ?? 'stream';
+    const response = await fetch(url, { headers });
+    createDebugMessages('embedjs:util:getSafe')(`URL '${url}' returned status code ${response.status}`);
+    if (response.status !== 200) throw new Error(`Failed to fetch URL '${url}'. Got status code ${response.status}.`);
+
+    return {
+        body:
+            format === 'text'
+                ? await response.text()
+                : format === 'buffer'
+                  ? Buffer.from(await response.arrayBuffer())
+                  : (response.body as unknown as NodeJS.ReadableStream),
+        statusCode: response.status,
+        headers: response.headers,
+    };
+}
diff --git a/core/embedjs/CHANGELOG.md b/core/embedjs/CHANGELOG.md
@@ -1,3 +1,13 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
+### 🩹 Fixes
+
+- renamed remaining instances if vectorDb to vectorDatabase ([ca79586](https://github.com/llm-tools/embedJs/commit/ca79586))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,12 +1,12 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.15",
-        "@llm-tools/embedjs-utils": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
+        "@llm-tools/embedjs-utils": "0.1.16",
         "debug": "^4.3.7",
         "langchain": "^0.3.5",
         "md5": "^2.3.0",
@@ -16,7 +16,7 @@
     "devDependencies": {
         "@types/debug": "^4.1.12",
         "@types/md5": "^2.3.5",
-        "@types/node": "^22.8.6"
+        "@types/node": "^22.8.7"
     },
     "main": "./src/index.js",
     "license": "Apache-2.0",

diff --git a/core/embedjs/src/loaders/json-loader.ts b/core/embedjs/src/loaders/json-loader.ts
@@ -48,7 +48,6 @@ export class JsonLoader extends BaseLoader<{ type: 'JsonLoader' }> {
                 metadata: {
                     type: 'JsonLoader' as const,
                     source: tuncatedObjectString,
-                    ...entry,
                 },
             };
         }

diff --git a/core/embedjs/src/loaders/url-loader.ts b/core/embedjs/src/loaders/url-loader.ts
@@ -2,7 +2,7 @@ import { getMimeType } from 'stream-mime-type';
 import createDebugMessages from 'debug';
 import md5 from 'md5';
 
-import { contentTypeToMimeType, truncateCenterString } from '@llm-tools/embedjs-utils';
+import { contentTypeToMimeType, getSafe, truncateCenterString } from '@llm-tools/embedjs-utils';
 import { BaseLoader } from '@llm-tools/embedjs-interfaces';
 import { createLoaderFromMimeType } from '../util/mime.js';
 
@@ -17,7 +17,7 @@ export class UrlLoader extends BaseLoader<{ type: 'UrlLoader' }> {
     }
 
     override async *getUnfilteredChunks() {
-        const response = await fetch(this.url, { headers: { 'Accept-Encoding': '' } });
+        const response = await getSafe(this.url.toString(), { headers: { 'Accept-Encoding': '' } });
         const stream = response.body as unknown as NodeJS.ReadableStream;
         let { mime } = await getMimeType(stream, { strict: true });
         this.debug(`Loader stream detected type '${mime}'`);

diff --git a/databases/embedjs-astra/CHANGELOG.md b/databases/embedjs-astra/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/CHANGELOG.md b/databases/embedjs-cosmos/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.1.1",
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/CHANGELOG.md b/databases/embedjs-hnswlib/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "debug": "^4.3.7",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/CHANGELOG.md b/databases/embedjs-lancedb/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.12.0",
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "compute-cosine-similarity": "^1.1.0"
     },
     "type": "module",

diff --git a/databases/embedjs-lmdb/CHANGELOG.md b/databases/embedjs-lmdb/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-lmdb/package.json b/databases/embedjs-lmdb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-lmdb",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add LMDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "lmdb": "^3.1.4"
     },
     "type": "module",

diff --git a/databases/embedjs-mongodb/CHANGELOG.md b/databases/embedjs-mongodb/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-mongodb/package.json b/databases/embedjs-mongodb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-mongodb",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add MongoDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "debug": "^4.3.7",
         "mongodb": "^6.10.0"
     },

diff --git a/databases/embedjs-pinecone/CHANGELOG.md b/databases/embedjs-pinecone/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-pinecone/package.json b/databases/embedjs-pinecone/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-pinecone",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add Pinecone support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "@pinecone-database/pinecone": "^4.0.0",
         "debug": "^4.3.7"
     },

diff --git a/databases/embedjs-pinecone/src/pinecone-db.ts b/databases/embedjs-pinecone/src/pinecone-db.ts
@@ -100,7 +100,7 @@ export class PineconeDb implements BaseVectorDatabase {
         } catch {
             this.debug(
                 `Failed to delete keys for loader '${uniqueLoaderId}'.
-Pinecone does not allow deleting by metadata filtering in serverless and free (what they call starter) instances`,
+Pinecone does not allow deleting by metadata filtering in serverless and free instances`,
             );
             return false;
         }

diff --git a/databases/embedjs-qdrant/CHANGELOG.md b/databases/embedjs-qdrant/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-qdrant/package.json b/databases/embedjs-qdrant/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-qdrant",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add Qdrant support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "@qdrant/js-client-rest": "^1.12.0",
         "debug": "^4.3.7",
         "uuid": "^11.0.2"

diff --git a/databases/embedjs-redis/CHANGELOG.md b/databases/embedjs-redis/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-redis/package.json b/databases/embedjs-redis/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-redis",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add Redis support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "ioredis": "^5.4.1"
     },
     "type": "module",

diff --git a/databases/embedjs-weaviate/CHANGELOG.md b/databases/embedjs-weaviate/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-weaviate/package.json b/databases/embedjs-weaviate/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-weaviate",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Add Weaviate support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-interfaces": "0.1.16",
         "compute-cosine-similarity": "^1.1.0",
         "debug": "^4.3.7",
         "weaviate-ts-client": "^2.2.0"