Merge pull request #178 from llm-tools/sqlite

LibSQL
llm-tools · Nov 27, 2024 · 1efd12e · 1efd12e
2 parents 1bb273e + 2140431
commit 1efd12e
Show file tree

Hide file tree

Showing 62 changed files with 688 additions and 189 deletions.
diff --git a/.gitignore b/.gitignore
@@ -64,6 +64,7 @@ acme.json
 temp/
 storage/
 lmdb/
+data.db
 
 #env
 .env
diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.21",
+    "version": "0.1.22",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
         "@langchain/core": "^0.3.19",

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.21",
+    "version": "0.1.22",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.21"
+        "@llm-tools/embedjs-interfaces": "0.1.22"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,12 +1,12 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.21",
+    "version": "0.1.22",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.21",
-        "@llm-tools/embedjs-utils": "0.1.21",
+        "@llm-tools/embedjs-interfaces": "0.1.22",
+        "@llm-tools/embedjs-utils": "0.1.22",
         "debug": "^4.3.7",
         "langchain": "^0.3.6",
         "md5": "^2.3.0",

diff --git a/core/embedjs/src/core/rag-application.ts b/core/embedjs/src/core/rag-application.ts
@@ -306,6 +306,17 @@ export class RAGApplication {
         return this.vectorDatabase.getVectorCount();
     }
 
+    /**
+     * The function `deleteConversation` deletes all entries related to a particular conversation from the database
+     * @param {string} conversationId - The `conversationId` that you want to delete. Pass 'default' to delete
+     * the default conversation thread that is created and maintained automatically
+     */
+    public async deleteConversation(conversationId: string) {
+        if (this.store) {
+            await this.store.deleteConversation(conversationId);
+        }
+    }
+
     /**
      * The function `deleteLoader` deletes embeddings from a loader after confirming the action.
      * @param {string} uniqueLoaderId - The `uniqueLoaderId` parameter is a string that represents the

diff --git a/core/embedjs/src/store/memory-store.ts b/core/embedjs/src/store/memory-store.ts
@@ -50,16 +50,22 @@ export class MemoryStore implements BaseStore {
         const loaderId = <string>this.loaderCustomValues[key].loaderId;
 
         delete this.loaderList[key];
-        this.loaderCustomValuesMap.set(
-            loaderId,
-            this.loaderCustomValuesMap.get(loaderId).filter((k) => k !== key),
-        );
+
+        if (this.loaderCustomValuesMap.has(loaderId)) {
+            this.loaderCustomValuesMap.set(
+                loaderId,
+                this.loaderCustomValuesMap.get(loaderId).filter((k) => k !== key),
+            );
+        }
     }
 
     async deleteLoaderMetadataAndCustomValues(loaderId: string): Promise<void> {
-        this.loaderCustomValuesMap.get(loaderId).forEach((key) => {
-            delete this.loaderCustomValues[key];
-        });
+        if (this.loaderCustomValuesMap.has(loaderId)) {
+            this.loaderCustomValuesMap.get(loaderId).forEach((key) => {
+                delete this.loaderCustomValues[key];
+            });
+        }
+
         this.loaderCustomValuesMap.delete(loaderId);
         delete this.loaderList[loaderId];
     }

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.21",
+    "version": "0.1.22",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.21",
+        "@llm-tools/embedjs-interfaces": "0.1.22",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.21",
+    "version": "0.1.22",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.2.0",
-        "@llm-tools/embedjs-interfaces": "0.1.21",
+        "@llm-tools/embedjs-interfaces": "0.1.22",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.21",
+    "version": "0.1.22",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.21",
+        "@llm-tools/embedjs-interfaces": "0.1.22",
         "debug": "^4.3.7",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.21",
+    "version": "0.1.22",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.13.0",
-        "@llm-tools/embedjs-interfaces": "0.1.21",
+        "@llm-tools/embedjs-interfaces": "0.1.22",
         "compute-cosine-similarity": "^1.1.0",
         "debug": "^4.3.7"
     },

diff --git a/databases/embedjs-libsql/CHANGELOG.md b/databases/embedjs-libsql/CHANGELOG.md
@@ -1,5 +1,5 @@
-## 0.1.19 (2024-11-26)
+## 0.1.22 (2024-11-26)
 
 ### 🚀 Features
 
--   added sqlite vector database ([cd27bc5](https://github.com/llm-tools/embedJs/commit/cd27bc5))
+- added sqlite vector database ([cd27bc5](https://github.com/llm-tools/embedJs/commit/cd27bc5))
diff --git a/databases/embedjs-libsql/package.json b/databases/embedjs-libsql/package.json
@@ -1,10 +1,11 @@
 {
     "name": "@llm-tools/embedjs-libsql",
-    "version": "0.1.21",
+    "version": "0.1.22",
     "description": "Add LibSQL support to embedjs",
     "dependencies": {
         "@libsql/client": "^0.14.0",
-        "@llm-tools/embedjs-interfaces": "0.1.21",
+        "@llm-tools/embedjs-interfaces": "0.1.22",
+        "@llm-tools/embedjs-utils": "0.1.22",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-libsql/src/index.ts b/databases/embedjs-libsql/src/index.ts
@@ -1 +1,2 @@
 export * from './libsql-db.js';
+export * from './libsql-store.js';
diff --git a/databases/embedjs-libsql/src/libsql-db.ts b/databases/embedjs-libsql/src/libsql-db.ts
@@ -2,6 +2,7 @@ import createDebugMessages from 'debug';
 import { Client, createClient } from '@libsql/client';
 
 import { BaseVectorDatabase, ExtractChunkData, InsertChunkData } from '@llm-tools/embedjs-interfaces';
+import { truncateCenterString } from '@llm-tools/embedjs-utils';
 
 export class LibSqlDb implements BaseVectorDatabase {
     private readonly debug = createDebugMessages('embedjs:vector:LibSqlDb');
@@ -16,46 +17,48 @@ export class LibSqlDb implements BaseVectorDatabase {
     }
 
     async init({ dimensions }: { dimensions: number }) {
-        await this.client.execute(`CREATE TABLE ${this.tableName} (
-            id              TEXT
-            pageContent     TEXT,
-            uniqueLoaderId  TEXT,
-            source          TEXT,
+        await this.client.execute(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
+            id              TEXT PRIMARY KEY,
+            pageContent     TEXT UNIQUE,
+            uniqueLoaderId  TEXT NOT NULL,
+            source          TEXT NOT NULL,
             vector          F32_BLOB(${dimensions}),
             metadata        TEXT
         );`);
     }
 
     async insertChunks(chunks: InsertChunkData[]): Promise<number> {
-        const values = chunks.map((chunk) => {
-            return `('${chunk.metadata.id}', '${chunk.pageContent}', '${chunk.metadata.uniqueLoaderId}', 
-                '${chunk.metadata.source}', vector32('[${chunk.vector.join(',')}]'),
-                '${JSON.stringify(chunk.metadata)}')`;
+        const batch = chunks.map((chunk) => {
+            return {
+                sql: `INSERT OR IGNORE INTO ${this.tableName} (id, pageContent, uniqueLoaderId, source, vector, metadata)
+            VALUES (?, ?, ?, ?, vector32('[${chunk.vector.join(',')}]'), ?);`,
+                args: [
+                    chunk.metadata.id,
+                    chunk.pageContent,
+                    chunk.metadata.uniqueLoaderId,
+                    chunk.metadata.source,
+                    JSON.stringify(chunk.metadata),
+                ],
+            };
         });
 
-        const statement = `INSERT INTO ${this.tableName} (id, pageContent, uniqueLoaderId, source, vector, metadata)
-            VALUES
-            ${values.join(',')}
-        );`;
-
-        this.debug(`Executing statement - ${statement}`);
-        const result = await this.client.execute(statement);
-        return result.rowsAffected;
+        this.debug(`Executing batch - ${truncateCenterString(JSON.stringify(batch), 1000)}`);
+        const result = await this.client.batch(batch, 'write');
+        return result.reduce((a, b) => a + b.rowsAffected, 0);
     }
 
     async similaritySearch(query: number[], k: number): Promise<ExtractChunkData[]> {
         const statement = `SELECT id, pageContent, uniqueLoaderId, source, metadata,
                 vector_distance_cos(vector, vector32('[${query.join(',')}]'))
             FROM ${this.tableName}
-            ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]'))
-            TOP ${k}
-            ASC;`;
+            ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]')) ASC
+            LIMIT ${k};`;
 
-        this.debug(`Executing statement - ${statement}`);
+        this.debug(`Executing statement - ${truncateCenterString(statement, 700)}`);
         const results = await this.client.execute(statement);
 
         return results.rows.map((result) => {
-            const metadata = JSON.parse(<string>result.metadata.toString());
+            const metadata = JSON.parse(result.metadata.toString());
 
             return {
                 metadata,
@@ -66,16 +69,21 @@ export class LibSqlDb implements BaseVectorDatabase {
     }
 
     async getVectorCount(): Promise<number> {
-        return 0;
+        const statement = `SELECT count(id) as count FROM ${this.tableName};`;
+
+        this.debug(`Executing statement - ${statement}`);
+        const results = await this.client.execute(statement);
+
+        return Number.parseInt(results.rows[0].count.toString());
     }
 
     async deleteKeys(uniqueLoaderId: string): Promise<boolean> {
-        const statement = `DELETE FROM ${this.tableName} VALUES WHERE
-           uniqueLoaderId = '${uniqueLoaderId}';`;
-
-        await this.client.execute(statement);
+        await this.client.execute(`DELETE FROM ${this.tableName} WHERE
+           uniqueLoaderId = '${uniqueLoaderId}';`);
         return true;
     }
 
-    async reset(): Promise<void> {}
+    async reset(): Promise<void> {
+        await this.client.execute(`DELETE FROM ${this.tableName};`);
+    }
 }