Merge pull request #115 from llm-tools/monorepo

Monorepo

core/embedjs-interfaces/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.1 (2024-10-04)
+
+Temporarily disabled dynamic, url and local path loaders as they required install of all modules from the monorepo. They will be reenabled soon.
+
 ## 0.1.0 (2024-10-03)
 
 This component has been extracted and is now published as part of a workspace monorepo managed by [NX](https://nx.dev/). There are many reasons that prompted this move, but the most critical issue was to decouple the need to install all dependencies for a single usecase. While we add (and continue to add) more and more loaders, databases, caches and models - the number of shared dependencies grew a lot. Most projects will not use all these combinations and it made no sense to have them all installed for everyone. Further, issues with dependent packages raised vulnerabilities that affected all projects - clearly something we did not intend.

core/embedjs-interfaces/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.0",
+    "version": "0.1.1",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
         "debug": "^4.3.7",

core/embedjs-utils/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.1 (2024-10-04)
+
+Temporarily disabled dynamic, url and local path loaders as they required install of all modules from the monorepo. They will be reenabled soon.
+
 ## 0.1.0 (2024-10-03)
 
 This component has been extracted and is now published as part of a workspace monorepo managed by [NX](https://nx.dev/). There are many reasons that prompted this move, but the most critical issue was to decouple the need to install all dependencies for a single usecase. While we add (and continue to add) more and more loaders, databases, caches and models - the number of shared dependencies grew a lot. Most projects will not use all these combinations and it made no sense to have them all installed for everyone. Further, issues with dependent packages raised vulnerabilities that affected all projects - clearly something we did not intend.

core/embedjs-utils/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.0",
+    "version": "0.1.1",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.0"
+        "@llm-tools/embedjs-interfaces": "0.1.1"
     },
     "type": "module",
     "main": "./src/index.js",

core/embedjs/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.1 (2024-10-04)
+
+Temporarily disabled dynamic, url and local path loaders as they required install of all modules from the monorepo. They will be reenabled soon.
+
 ## 0.1.0 (2024-10-03)
 
 This component has been extracted and is now published as part of a workspace monorepo managed by [NX](https://nx.dev/). There are many reasons that prompted this move, but the most critical issue was to decouple the need to install all dependencies for a single usecase. While we add (and continue to add) more and more loaders, databases, caches and models - the number of shared dependencies grew a lot. Most projects will not use all these combinations and it made no sense to have them all installed for everyone. Further, issues with dependent packages raised vulnerabilities that affected all projects - clearly something we did not intend.

core/embedjs/package.json

@@ -1,11 +1,11 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.0",
+    "version": "0.1.1",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.0",
-        "@llm-tools/embedjs-utils": "0.1.0",
+        "@llm-tools/embedjs-interfaces": "0.1.1",
+        "@llm-tools/embedjs-utils": "0.1.1",
         "axios": "^1.7.7",
         "debug": "^4.3.7",
         "langchain": "^0.3.2",

core/embedjs/src/core/rag-application-builder.ts

@@ -1,14 +1,12 @@
 import { RAGApplication } from './rag-application.js';
-import { LoaderParam } from './dynamic-loader-selector.js';
 import {
     BaseCache,
     BaseConversation,
     BaseDb,
     BaseEmbeddings,
+    BaseLoader,
     BaseModel,
-    SIMPLE_MODELS,
 } from '@llm-tools/embedjs-interfaces';
-import { OpenAi } from '@llm-tools/embedjs-openai';
 
 export class RAGApplicationBuilder {
     private temperature: number;
@@ -18,7 +16,7 @@ export class RAGApplicationBuilder {
     private searchResultCount: number;
     private embeddingModel: BaseEmbeddings;
     private embeddingRelevanceCutOff: number;
-    private loaders: LoaderParam[];
+    private loaders: BaseLoader[];
     private vectorDb: BaseDb;
     private conversations: BaseConversation;
 
@@ -32,7 +30,6 @@ export class RAGApplicationBuilder {
 
         Do not use words like context or training data when responding. You can say you do not have all the information but do not indicate that you are not a reliable source.`;
 
-        this.setModel(SIMPLE_MODELS.OPENAI_GPT4_O);
         this.embeddingRelevanceCutOff = 0;
     }
 
@@ -42,7 +39,7 @@ export class RAGApplicationBuilder {
         return entity;
     }
 
-    addLoader(loader: LoaderParam) {
+    addLoader(loader: BaseLoader) {
         this.loaders.push(loader);
         return this;
     }
@@ -86,17 +83,9 @@ export class RAGApplicationBuilder {
         return this;
     }
 
-    setModel(model: 'NO_MODEL' | SIMPLE_MODELS | BaseModel) {
+    setModel(model: 'NO_MODEL' | BaseModel) {
         if (typeof model === 'object') this.model = model;
-        else {
-            if (model === SIMPLE_MODELS.OPENAI_GPT4_O) this.model = new OpenAi({ modelName: 'gpt-4o' });
-            else if (model === SIMPLE_MODELS['OPENAI_GPT4_TURBO'])
-                this.model = new OpenAi({ modelName: 'gpt-4-turbo' });
-            else if (model === SIMPLE_MODELS['OPENAI_GPT3.5_TURBO'])
-                this.model = new OpenAi({ modelName: 'gpt-3.5-turbo' });
-            else this.model = null;
-        }
-
+        else this.model = null;
         return this;
     }
 

core/embedjs/src/core/rag-application.ts

@@ -1,8 +1,7 @@
 import createDebugMessages from 'debug';
 
-import { DynamicLoader, LoaderParam } from './dynamic-loader-selector.js';
-import { RAGApplicationBuilder } from './rag-application-builder.js';
 import { RAGEmbedding } from './rag-embedding.js';
+import { RAGApplicationBuilder } from './rag-application-builder.js';
 import { InMemoryConversation } from '../conversation/memory-conversations.js';
 import {
     AddLoaderReturn,
@@ -17,7 +16,6 @@ import {
     QueryResponse,
 } from '@llm-tools/embedjs-interfaces';
 import { cleanString, getUnique } from '@llm-tools/embedjs-utils';
-import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
 
 export class RAGApplication {
     private readonly debug = createDebugMessages('embedjs:core');
@@ -28,7 +26,7 @@ export class RAGApplication {
     private readonly model: BaseModel;
     private readonly embeddingRelevanceCutOff: number;
 
-    private readonly rawLoaders: LoaderParam[];
+    private readonly rawLoaders: BaseLoader[];
     private loaders: BaseLoader[];
 
     constructor(llmBuilder: RAGApplicationBuilder) {
@@ -52,7 +50,7 @@ export class RAGApplication {
         this.searchResultCount = llmBuilder.getSearchResultCount();
         this.embeddingRelevanceCutOff = llmBuilder.getEmbeddingRelevanceCutOff();
 
-        RAGEmbedding.init(llmBuilder.getEmbeddingModel() ?? new OpenAi3SmallEmbeddings());
+        RAGEmbedding.init(llmBuilder.getEmbeddingModel());
     }
 
     /**
@@ -84,7 +82,7 @@ export class RAGApplication {
      * cache, and pre-loaders.
      */
     public async init() {
-        this.loaders = await DynamicLoader.createLoaders(this.rawLoaders);
+        this.loaders = this.rawLoaders;
 
         if (this.model) {
             await this.model.init();
@@ -116,9 +114,8 @@ export class RAGApplication {
      * - `uniqueId`: Unique identifier of the loader
      * - `loaderType`: Name of the loader's constructor class
      */
-    public async addLoader(loaderParam: LoaderParam): Promise<AddLoaderReturn> {
-        const loader = await DynamicLoader.createLoader(loaderParam);
-        return this._addLoader(loader);
+    public async addLoader(loaderParam: BaseLoader): Promise<AddLoaderReturn> {
+        return this._addLoader(loaderParam);
     }
 
     /**

core/embedjs/src/index.ts

@@ -2,8 +2,5 @@ import { RAGApplication } from './core/rag-application.js';
 import { RAGApplicationBuilder } from './core/rag-application-builder.js';
 import { TextLoader } from './loaders/text-loader.js';
 import { JsonLoader } from './loaders/json-loader.js';
-import { LocalPathLoader } from './loaders/local-path-loader.js';
-import { UrlLoader } from './loaders/url-loader.js';
-import { SIMPLE_MODELS } from '@llm-tools/embedjs-interfaces';
 
-export { RAGApplication, RAGApplicationBuilder, TextLoader, JsonLoader, UrlLoader, LocalPathLoader, SIMPLE_MODELS };
+export { RAGApplication, RAGApplicationBuilder, TextLoader, JsonLoader };