Merge pull request #173 from llm-tools/sqlite

Sqlite
llm-tools · Nov 26, 2024 · 60b4a4c · 60b4a4c
2 parents 574dc62 + 11ceaf2
commit 60b4a4c
Show file tree

Hide file tree

Showing 75 changed files with 928 additions and 1,161 deletions.
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -1,3 +1,3 @@
 {
-    "recommendations": ["nrwl.angular-console", "esbenp.prettier-vscode"]
+    "recommendations": ["esbenp.prettier-vscode"]
 }
diff --git a/core/embedjs-interfaces/CHANGELOG.md b/core/embedjs-interfaces/CHANGELOG.md
@@ -1,7 +1,11 @@
-## 0.1.18 (2024-11-13)
-
-This was a version bump only for embedjs-interfaces to align it with other projects, there were no code changes.
-
+## 0.1.19 (2024-11-26)
+
+This was a version bump only for this package to align it with other projects, there were no code changes.
+
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-interfaces to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.18",
+    "version": "0.1.19",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
         "@langchain/core": "^0.3.18",

diff --git a/core/embedjs-utils/CHANGELOG.md b/core/embedjs-utils/CHANGELOG.md
@@ -1,7 +1,11 @@
-## 0.1.18 (2024-11-13)
-
-This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
-
+## 0.1.19 (2024-11-26)
+
+This was a version bump only for this package to align it with other projects, there were no code changes.
+
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.18",
+    "version": "0.1.19",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.18"
+        "@llm-tools/embedjs-interfaces": "0.1.19"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs/CHANGELOG.md b/core/embedjs/CHANGELOG.md
@@ -1,7 +1,11 @@
-## 0.1.18 (2024-11-13)
-
-This was a version bump only for embedjs to align it with other projects, there were no code changes.
-
+## 0.1.19 (2024-11-26)
+
+This was a version bump only for this package to align it with other projects, there were no code changes.
+
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,12 +1,12 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.18",
+    "version": "0.1.19",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.18",
-        "@llm-tools/embedjs-utils": "0.1.18",
+        "@llm-tools/embedjs-interfaces": "0.1.19",
+        "@llm-tools/embedjs-utils": "0.1.19",
         "debug": "^4.3.7",
         "langchain": "^0.3.6",
         "md5": "^2.3.0",

diff --git a/databases/embedjs-astra/CHANGELOG.md b/databases/embedjs-astra/CHANGELOG.md
@@ -1,7 +1,11 @@
-## 0.1.18 (2024-11-13)
-
-This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
-
+## 0.1.19 (2024-11-26)
+
+This was a version bump only for this package to align it with other projects, there were no code changes.
+
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.18",
+    "version": "0.1.19",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.18",
+        "@llm-tools/embedjs-interfaces": "0.1.19",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/CHANGELOG.md b/databases/embedjs-cosmos/CHANGELOG.md
@@ -1,7 +1,11 @@
-## 0.1.18 (2024-11-13)
-
-This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
-
+## 0.1.19 (2024-11-26)
+
+This was a version bump only for this package to align it with other projects, there were no code changes.
+
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.18",
+    "version": "0.1.19",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.2.0",
-        "@llm-tools/embedjs-interfaces": "0.1.18",
+        "@llm-tools/embedjs-interfaces": "0.1.19",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/CHANGELOG.md b/databases/embedjs-hnswlib/CHANGELOG.md
@@ -1,7 +1,11 @@
-## 0.1.18 (2024-11-13)
-
-This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
-
+## 0.1.19 (2024-11-26)
+
+This was a version bump only for this package to align it with other projects, there were no code changes.
+
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.18",
+    "version": "0.1.19",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.18",
+        "@llm-tools/embedjs-interfaces": "0.1.19",
         "debug": "^4.3.7",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/CHANGELOG.md b/databases/embedjs-lancedb/CHANGELOG.md
@@ -1,7 +1,11 @@
-## 0.1.18 (2024-11-13)
-
-This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
-
+## 0.1.19 (2024-11-26)
+
+This was a version bump only for this package to align it with other projects, there were no code changes.
+
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,11 +1,12 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.18",
+    "version": "0.1.19",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.13.0",
-        "@llm-tools/embedjs-interfaces": "0.1.18",
-        "compute-cosine-similarity": "^1.1.0"
+        "@llm-tools/embedjs-interfaces": "0.1.19",
+        "compute-cosine-similarity": "^1.1.0",
+        "debug": "^4.3.7"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/databases/embedjs-lancedb/src/lance-db.ts b/databases/embedjs-lancedb/src/lance-db.ts
@@ -1,11 +1,13 @@
 import * as fsOld from 'node:fs';
 import * as fs from 'node:fs/promises';
+import createDebugMessages from 'debug';
 import similarity from 'compute-cosine-similarity';
 import { Table, connect } from '@lancedb/lancedb';
 
 import { BaseVectorDatabase, ExtractChunkData, InsertChunkData } from '@llm-tools/embedjs-interfaces';
 
 export class LanceDb implements BaseVectorDatabase {
+    private readonly debug = createDebugMessages('embedjs:vector:LanceDb');
     private static readonly STATIC_DB_NAME = 'vectors';
     private readonly isTemp: boolean = true;
     private readonly path: string;
@@ -18,13 +20,16 @@ export class LanceDb implements BaseVectorDatabase {
 
     async init({ dimensions }: { dimensions: number }) {
         if (!this.isTemp && !fsOld.existsSync(this.path)) {
+            this.debug(`Creating dir at path - ${this.path}`);
             await fs.mkdir(this.path);
         }
 
         const dir = await (this.isTemp ? fs.mkdtemp(this.path) : this.path);
+        this.debug(`Connecting to path - ${dir}`);
         const client = await connect(dir);
 
         const list = await client.tableNames();
+        this.debug.log(`Table names found - [${list.join(',')}]`);
         if (list.indexOf(LanceDb.STATIC_DB_NAME) > -1) this.table = await client.openTable(LanceDb.STATIC_DB_NAME);
         else {
             //TODO: You can add a proper schema instead of a sample record now but it requires another package apache-arrow; another install on downstream as well
@@ -56,12 +61,14 @@ export class LanceDb implements BaseVectorDatabase {
             };
         });
 
+        this.debug.log(`Executing insert of ${mapped.length} entries`);
         await this.table.add(mapped);
         return mapped.length; //TODO: check if vectorDb has addressed the issue where add returns undefined
     }
 
     async similaritySearch(query: number[], k: number): Promise<ExtractChunkData[]> {
         const results = await this.table.search(query).limit(k).toArray();
+        this.debug.log(`Query found ${results.length} entries`);
 
         return (
             results

diff --git a/databases/embedjs-libsql/CHANGELOG.md b/databases/embedjs-libsql/CHANGELOG.md
@@ -0,0 +1,5 @@
+## 0.1.19 (2024-11-26)
+
+### 🚀 Features
+
+-   added sqlite vector database ([cd27bc5](https://github.com/llm-tools/embedJs/commit/cd27bc5))
diff --git a/databases/embedjs-libsql/README.md b/databases/embedjs-libsql/README.md
@@ -0,0 +1,8 @@
+# embedjs-libsql
+
+<p>
+<a href="https://www.npmjs.com/package/@llm-tools/embedjs"  target="_blank"><img alt="NPM Version" src="https://img.shields.io/npm/v/%40llm-tools/embedjs?style=for-the-badge"></a>
+<a href="https://www.npmjs.com/package/@llm-tools/embedjs"  target="_blank"><img alt="License" src="https://img.shields.io/npm/l/%40llm-tools%2Fembedjs?style=for-the-badge"></a>
+</p>
+
+This package extends and offers additional functionality to [embedJs](https://www.npmjs.com/package/@llm-tools/embedjs). Refer to the documentation there for more details.
diff --git a/databases/embedjs-libsql/eslint.config.js b/databases/embedjs-libsql/eslint.config.js
@@ -0,0 +1,20 @@
+import baseConfig from '../../eslint.config.js';
+import parser from '@nx/eslint-plugin';
+
+export default [
+    ...baseConfig,
+    {
+        files: ['**/*.json'],
+        rules: {
+            '@nx/dependency-checks': [
+                'error',
+                {
+                    ignoredFiles: ['{projectRoot}/eslint.config.{js,cjs,mjs}'],
+                },
+            ],
+        },
+        languageOptions: {
+            parser,
+        },
+    },
+];
diff --git a/databases/embedjs-libsql/package.json b/databases/embedjs-libsql/package.json
@@ -0,0 +1,39 @@
+{
+    "name": "@llm-tools/embedjs-libsql",
+    "version": "0.1.19",
+    "description": "Add LibSQL support to embedjs",
+    "dependencies": {
+        "@libsql/client": "^0.14.0",
+        "@llm-tools/embedjs-interfaces": "0.1.19",
+        "debug": "^4.3.7"
+    },
+    "type": "module",
+    "main": "./src/index.js",
+    "license": "Apache-2.0",
+    "publishConfig": {
+        "access": "public"
+    },
+    "keywords": [
+        "llm",
+        "ai",
+        "gpt3",
+        "chain",
+        "prompt",
+        "prompt engineering",
+        "chatgpt",
+        "machine learning",
+        "ml",
+        "anthropic",
+        "embeddings",
+        "vectorstores"
+    ],
+    "author": "K V Adhityan",
+    "bugs": {
+        "url": "https://github.com/llm-tools/embedjs/issues"
+    },
+    "homepage": "https://github.com/llm-tools/embedjs#readme",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/llm-tools/embedjs.git"
+    }
+}
diff --git a/databases/embedjs-libsql/project.json b/databases/embedjs-libsql/project.json
@@ -0,0 +1,29 @@
+{
+    "name": "embedjs-libsql",
+    "$schema": "../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "databases/embedjs-libsql/src",
+    "projectType": "library",
+    "tags": [],
+    "targets": {
+        "build": {
+            "executor": "@nx/js:tsc",
+            "outputs": ["{options.outputPath}"],
+            "options": {
+                "outputPath": "dist/esm/embedjs-libsql",
+                "main": "databases/embedjs-libsql/src/index.ts",
+                "tsConfig": "databases/embedjs-libsql/tsconfig.lib.json",
+                "assets": ["databases/embedjs-libsql/*.md"]
+            }
+        },
+        "build-cjs": {
+            "executor": "@nx/js:tsc",
+            "outputs": ["{options.outputPath}"],
+            "dependsOn": ["^build-cjs"],
+            "options": {
+                "outputPath": "dist/cjs/embedjs-libsql",
+                "main": "databases/embedjs-libsql/src/index.ts",
+                "tsConfig": "databases/embedjs-libsql/tsconfig.cjs.json"
+            }
+        }
+    }
+}
diff --git a/databases/embedjs-libsql/src/index.ts b/databases/embedjs-libsql/src/index.ts
@@ -0,0 +1 @@
+export * from './libsql-db.js';