Merge pull request #156 from llm-tools/loaders

Added Markdown loader
llm-tools · Nov 5, 2024 · df0bfa7 · df0bfa7
2 parents 73cd1e7 + d17e3e3
commit df0bfa7
Show file tree

Hide file tree

Showing 91 changed files with 1,633 additions and 321 deletions.
diff --git a/core/embedjs-interfaces/CHANGELOG.md b/core/embedjs-interfaces/CHANGELOG.md
@@ -1,9 +1,15 @@
-## 0.1.16 (2024-11-04)
-
-### 🚀 Features
-
-- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
-
+## 0.1.17 (2024-11-05)
+
+### 🚀 Features
+
+-   added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d)
+
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+-   added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.16",
+    "version": "0.1.17",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
         "@langchain/core": "^0.3.17",

diff --git a/core/embedjs-utils/CHANGELOG.md b/core/embedjs-utils/CHANGELOG.md
@@ -1,9 +1,15 @@
-## 0.1.16 (2024-11-04)
-
-### 🚀 Features
-
-- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
-
+## 0.1.17 (2024-11-05)
+
+### 🚀 Features
+
+-   added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d)
+
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+-   added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.16",
+    "version": "0.1.17",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.16"
+        "@llm-tools/embedjs-interfaces": "0.1.17"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs/CHANGELOG.md b/core/embedjs/CHANGELOG.md
@@ -1,13 +1,19 @@
-## 0.1.16 (2024-11-04)
-
-### 🚀 Features
-
-- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
-
-### 🩹 Fixes
-
-- renamed remaining instances if vectorDb to vectorDatabase ([ca79586](https://github.com/llm-tools/embedJs/commit/ca79586))
-
+## 0.1.17 (2024-11-05)
+
+### 🚀 Features
+
+-   added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d)
+
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+-   added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
+### 🩹 Fixes
+
+-   renamed remaining instances if vectorDb to vectorDatabase ([ca79586](https://github.com/llm-tools/embedJs/commit/ca79586))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,12 +1,12 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.16",
+    "version": "0.1.17",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.16",
-        "@llm-tools/embedjs-utils": "0.1.16",
+        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-utils": "0.1.17",
         "debug": "^4.3.7",
         "langchain": "^0.3.5",
         "md5": "^2.3.0",

diff --git a/core/embedjs/src/core/rag-application-builder.ts b/core/embedjs/src/core/rag-application-builder.ts
@@ -24,7 +24,7 @@ export class RAGApplicationBuilder {
     constructor() {
         this.loaders = [];
         this.temperature = 0.1;
-        this.searchResultCount = 7;
+        this.searchResultCount = 30;
         this.model = SIMPLE_MODELS.OPENAI_GPT4_TURBO;
 
         this.systemMessage = `You are a helpful human like chat bot. Use relevant provided context and chat history to answer the query at the end. Answer in full.

diff --git a/core/embedjs/src/loaders/url-loader.ts b/core/embedjs/src/loaders/url-loader.ts
@@ -17,7 +17,7 @@ export class UrlLoader extends BaseLoader<{ type: 'UrlLoader' }> {
     }
 
     override async *getUnfilteredChunks() {
-        const response = await getSafe(this.url.toString(), { headers: { 'Accept-Encoding': '' } });
+        const response = await getSafe(this.url.href, { headers: { 'Accept-Encoding': '' } });
         const stream = response.body as unknown as NodeJS.ReadableStream;
         let { mime } = await getMimeType(stream, { strict: true });
         this.debug(`Loader stream detected type '${mime}'`);

diff --git a/core/embedjs/src/util/mime.ts b/core/embedjs/src/util/mime.ts
@@ -3,7 +3,7 @@ import createDebugMessages from 'debug';
 import { BaseLoader } from '@llm-tools/embedjs-interfaces';
 import { TextLoader } from '../loaders/text-loader.js';
 
-export async function createLoaderFromMimeType(loader: string, mimeType: string): Promise<BaseLoader> {
+export async function createLoaderFromMimeType(loaderData: string, mimeType: string): Promise<BaseLoader> {
     createDebugMessages('embedjs:util:createLoaderFromMimeType')(`Incoming mime type '${mimeType}'`);
 
     switch (mimeType) {
@@ -15,7 +15,7 @@ export async function createLoaderFromMimeType(loader: string, mimeType: string)
                 );
             });
             createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported DocxLoader');
-            return new DocxLoader({ filePathOrUrl: loader });
+            return new DocxLoader({ filePathOrUrl: loaderData });
         }
         case 'application/vnd.ms-excel':
         case 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': {
@@ -25,14 +25,14 @@ export async function createLoaderFromMimeType(loader: string, mimeType: string)
                 );
             });
             createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported ExcelLoader');
-            return new ExcelLoader({ filePathOrUrl: loader });
+            return new ExcelLoader({ filePathOrUrl: loaderData });
         }
         case 'application/pdf': {
             const { PdfLoader } = await import('@llm-tools/embedjs-loader-pdf').catch(() => {
                 throw new Error('Package `@llm-tools/embedjs-loader-pdf` needs to be installed to load PDF files');
             });
             createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported PdfLoader');
-            return new PdfLoader({ filePathOrUrl: loader });
+            return new PdfLoader({ filePathOrUrl: loaderData });
         }
         case 'application/vnd.openxmlformats-officedocument.presentationml.presentation': {
             const { PptLoader } = await import('@llm-tools/embedjs-loader-msoffice').catch(() => {
@@ -41,44 +41,52 @@ export async function createLoaderFromMimeType(loader: string, mimeType: string)
                 );
             });
             createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported PptLoader');
-            return new PptLoader({ filePathOrUrl: loader });
+            return new PptLoader({ filePathOrUrl: loaderData });
         }
         case 'text/plain': {
-            const fineType = mime.getType(loader);
-            createDebugMessages('embedjs:util:createLoaderFromMimeType')(`Fine type for '${loader}' is '${fineType}'`);
+            const fineType = mime.getType(loaderData);
+            createDebugMessages('embedjs:util:createLoaderFromMimeType')(
+                `Fine type for '${loaderData}' is '${fineType}'`,
+            );
             if (fineType === 'text/csv') {
                 const { CsvLoader } = await import('@llm-tools/embedjs-loader-csv').catch(() => {
-                    throw new Error('Package `@llm-tools/embedjs-loader-csv` needs to be installed to load csv files');
+                    throw new Error('Package `@llm-tools/embedjs-loader-csv` needs to be installed to load CSV files');
                 });
 
                 createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported CsvLoader');
-                return new CsvLoader({ filePathOrUrl: loader });
-            } else return new TextLoader({ text: loader });
+                return new CsvLoader({ filePathOrUrl: loaderData });
+            } else return new TextLoader({ text: loaderData });
         }
         case 'application/csv': {
             const { CsvLoader } = await import('@llm-tools/embedjs-loader-csv').catch(() => {
-                throw new Error('Package `@llm-tools/embedjs-loader-csv` needs to be installed to load csv files');
+                throw new Error('Package `@llm-tools/embedjs-loader-csv` needs to be installed to load CSV files');
             });
             createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported CsvLoader');
-            return new CsvLoader({ filePathOrUrl: loader });
+            return new CsvLoader({ filePathOrUrl: loaderData });
         }
         case 'text/html': {
             const { WebLoader } = await import('@llm-tools/embedjs-loader-web').catch(() => {
                 throw new Error('Package `@llm-tools/embedjs-loader-web` needs to be installed to load web documents');
             });
             createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported WebLoader');
-            return new WebLoader({ urlOrContent: loader });
+            return new WebLoader({ urlOrContent: loaderData });
         }
         case 'text/xml': {
             const { SitemapLoader } = await import('@llm-tools/embedjs-loader-sitemap').catch(() => {
                 throw new Error('Package `@llm-tools/embedjs-loader-sitemap` needs to be installed to load sitemaps');
             });
             createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported SitemapLoader');
 
-            if (await SitemapLoader.test(loader)) {
-                return new SitemapLoader({ url: loader });
+            if (await SitemapLoader.test(loaderData)) {
+                return new SitemapLoader({ url: loaderData });
             }
-            throw new Error(`No loader supported for generic xml`);
+
+            //This is not a Sitemap but is still XML
+            const { XmlLoader } = await import('@llm-tools/embedjs-loader-xml').catch(() => {
+                throw new Error('Package `@llm-tools/embedjs-loader-xml` needs to be installed to load XML documents');
+            });
+            createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported XmlLoader');
+            return new XmlLoader({ filePathOrUrl: loaderData });
         }
         case undefined:
             throw new Error(`MIME type could not be detected. Please file an issue if you think this is a bug.`);

diff --git a/databases/embedjs-astra/CHANGELOG.md b/databases/embedjs-astra/CHANGELOG.md
@@ -1,9 +1,15 @@
-## 0.1.16 (2024-11-04)
-
-### 🚀 Features
-
-- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
-
+## 0.1.17 (2024-11-05)
+
+### 🚀 Features
+
+-   added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d)
+
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+-   added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.16",
+    "version": "0.1.17",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.16",
+        "@llm-tools/embedjs-interfaces": "0.1.17",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/CHANGELOG.md b/databases/embedjs-cosmos/CHANGELOG.md
@@ -1,9 +1,15 @@
-## 0.1.16 (2024-11-04)
-
-### 🚀 Features
-
-- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
-
+## 0.1.17 (2024-11-05)
+
+### 🚀 Features
+
+-   added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d)
+
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+-   added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.16",
+    "version": "0.1.17",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.1.1",
-        "@llm-tools/embedjs-interfaces": "0.1.16",
+        "@llm-tools/embedjs-interfaces": "0.1.17",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/CHANGELOG.md b/databases/embedjs-hnswlib/CHANGELOG.md
@@ -1,9 +1,15 @@
-## 0.1.16 (2024-11-04)
-
-### 🚀 Features
-
-- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
-
+## 0.1.17 (2024-11-05)
+
+### 🚀 Features
+
+-   added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d)
+
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+-   added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.16",
+    "version": "0.1.17",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.16",
+        "@llm-tools/embedjs-interfaces": "0.1.17",
         "debug": "^4.3.7",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/CHANGELOG.md b/databases/embedjs-lancedb/CHANGELOG.md
@@ -1,9 +1,15 @@
-## 0.1.16 (2024-11-04)
-
-### 🚀 Features
-
-- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
-
+## 0.1.17 (2024-11-05)
+
+### 🚀 Features
+
+-   added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d)
+
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+-   added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.16",
+    "version": "0.1.17",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.12.0",
-        "@llm-tools/embedjs-interfaces": "0.1.16",
+        "@llm-tools/embedjs-interfaces": "0.1.17",
         "compute-cosine-similarity": "^1.1.0"
     },
     "type": "module",

diff --git a/databases/embedjs-lmdb/CHANGELOG.md b/databases/embedjs-lmdb/CHANGELOG.md
@@ -1,9 +1,15 @@
-## 0.1.16 (2024-11-04)
-
-### 🚀 Features
-
-- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
-
+## 0.1.17 (2024-11-05)
+
+### 🚀 Features
+
+-   added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d)
+
+## 0.1.16 (2024-11-04)
+
+### 🚀 Features
+
+-   added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511))
+
 ## 0.1.15 and 0.1.14 (2024-11-01)
 
 ### 🚀 Features