Skip to content

Commit

Permalink
Merge pull request #177 from llm-tools/sqlite
Browse files Browse the repository at this point in the history 
LanceDb
  • Loading branch information
@adhityan
adhityan authored Nov 26, 2024
2 parents 53b5fe1 + c1898e9 commit 1bb273e
Show file tree
Hide file tree
Showing 35 changed files with 400 additions and 515 deletions.
4 changes: 2 additions & 2 deletions core/embedjs-interfaces/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-interfaces",
"version": "0.1.20",
"version": "0.1.21",
"description": "Interfaces for extending the embedjs ecosystem",
"dependencies": {
"@langchain/core": "^0.3.18",
"@langchain/core": "^0.3.19",
"debug": "^4.3.7",
"md5": "^2.3.0",
"uuid": "^11.0.3"
Expand Down
4 changes: 2 additions & 2 deletions core/embedjs-utils/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-utils",
"version": "0.1.20",
"version": "0.1.21",
"description": "Useful util functions when extending the embedjs ecosystem",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20"
"@llm-tools/embedjs-interfaces": "0.1.21"
},
"type": "module",
"main": "./src/index.js",
Expand Down
8 changes: 4 additions & 4 deletions core/embedjs/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
{
"type": "module",
"name": "@llm-tools/embedjs",
"version": "0.1.20",
"version": "0.1.21",
"description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
"dependencies": {
"@langchain/textsplitters": "^0.1.0",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-utils": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@llm-tools/embedjs-utils": "0.1.21",
"debug": "^4.3.7",
"langchain": "^0.3.6",
"md5": "^2.3.0",
Expand All @@ -16,7 +16,7 @@
"devDependencies": {
"@types/debug": "^4.1.12",
"@types/md5": "^2.3.5",
"@types/node": "^22.9.1"
"@types/node": "^22.10.0"
},
"main": "./src/index.js",
"license": "Apache-2.0",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-astra/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-astradb",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add AstraDB support to embedjs",
"dependencies": {
"@datastax/astra-db-ts": "^1.5.0",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"debug": "^4.3.7"
},
"type": "module",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-cosmos/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-cosmos",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add CosmosDB support to embedjs",
"dependencies": {
"@azure/cosmos": "^4.2.0",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"debug": "^4.3.7"
},
"type": "module",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-hnswlib/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-hnswlib",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add HNSWLib support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"debug": "^4.3.7",
"hnswlib-node": "^3.0.0"
},
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-lancedb/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-lancedb",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add LanceDb support to embedjs",
"dependencies": {
"@lancedb/lancedb": "^0.13.0",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"compute-cosine-similarity": "^1.1.0",
"debug": "^4.3.7"
},
Expand Down
19 changes: 10 additions & 9 deletions databases/embedjs-lancedb/src/lance-db.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,14 @@ export class LanceDb implements BaseVectorDatabase {
}

const dir = await (this.isTemp ? fs.mkdtemp(this.path) : this.path);
this.debug(`Connecting to path - ${dir}`);
this.debug(`Connecting to database at path - ${dir}`);
const client = await connect(dir);

const list = await client.tableNames();
this.debug(`Table names found - [${list.join(',')}]`);
if (list.indexOf(LanceDb.STATIC_DB_NAME) > -1) this.table = await client.openTable(LanceDb.STATIC_DB_NAME);
else {
//TODO: You can add a proper schema instead of a sample record now but it requires another package apache-arrow; another install on downstream as well
this.table = await client.createTable(LanceDb.STATIC_DB_NAME, [
this.debug('Trying to connect to table');
this.table = await client.createTable(
LanceDb.STATIC_DB_NAME,
[
//TODO: You can add a proper schema instead of a sample record now but it requires another package apache-arrow; another install on downstream as well
{
id: 'md5',
pageContent: 'sample',
Expand All @@ -42,8 +41,10 @@ export class LanceDb implements BaseVectorDatabase {
vectorString: 'sample',
metadata: 'sample',
},
]);
}
],
{ existOk: true },
);
this.debug('Connected to table');
}

async insertChunks(chunks: InsertChunkData[]): Promise<number> {
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-libsql/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-libsql",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add LibSQL support to embedjs",
"dependencies": {
"@libsql/client": "^0.14.0",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"debug": "^4.3.7"
},
"type": "module",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-lmdb/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-lmdb",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add LMDB support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"debug": "^4.3.7",
"lmdb": "^3.1.6"
},
Expand Down
6 changes: 3 additions & 3 deletions databases/embedjs-mongodb/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "@llm-tools/embedjs-mongodb",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add MongoDB support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"debug": "^4.3.7",
"mongodb": "^6.10.0"
"mongodb": "^6.11.0"
},
"type": "module",
"main": "./src/index.js",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-pinecone/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-pinecone",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add Pinecone support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@pinecone-database/pinecone": "^4.0.0",
"debug": "^4.3.7"
},
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-qdrant/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-qdrant",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add Qdrant support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@qdrant/js-client-rest": "^1.12.0",
"debug": "^4.3.7",
"uuid": "^11.0.3"
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-redis/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-redis",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add Redis support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"ioredis": "^5.4.1"
},
"type": "module",
Expand Down
4 changes: 2 additions & 2 deletions databases/embedjs-weaviate/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-weaviate",
"version": "0.1.20",
"version": "0.1.21",
"description": "Add Weaviate support to embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"compute-cosine-similarity": "^1.1.0",
"debug": "^4.3.7",
"weaviate-ts-client": "^2.2.0"
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-confluence/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-loader-confluence",
"version": "0.1.20",
"version": "0.1.21",
"description": "Confluence loader for embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-loader-web": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@llm-tools/embedjs-loader-web": "0.1.21",
"confluence.js": "^1.7.4",
"debug": "^4.3.7",
"md5": "^2.3.0"
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-csv/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-loader-csv",
"version": "0.1.20",
"version": "0.1.21",
"description": "CSV loader for embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-utils": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@llm-tools/embedjs-utils": "0.1.21",
"csv-parse": "^5.6.0",
"debug": "^4.3.7",
"md5": "^2.3.0"
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-markdown/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-loader-markdown",
"version": "0.1.20",
"version": "0.1.21",
"description": "XML loader for embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-loader-web": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@llm-tools/embedjs-loader-web": "0.1.21",
"debug": "^4.3.7",
"md5": "^2.3.0",
"micromark": "^4.0.1",
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-msoffice/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "@llm-tools/embedjs-loader-msoffice",
"version": "0.1.20",
"version": "0.1.21",
"description": "Word, PPT and Excel loader for embedjs",
"dependencies": {
"@langchain/textsplitters": "^0.1.0",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-utils": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@llm-tools/embedjs-utils": "0.1.21",
"md5": "^2.3.0",
"office-text-extractor": "^3.0.3"
},
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-pdf/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "@llm-tools/embedjs-loader-pdf",
"version": "0.1.20",
"version": "0.1.21",
"description": "PDF loader for embedjs",
"dependencies": {
"@langchain/textsplitters": "^0.1.0",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-utils": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@llm-tools/embedjs-utils": "0.1.21",
"md5": "^2.3.0",
"office-text-extractor": "^3.0.3"
},
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-sitemap/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-loader-sitemap",
"version": "0.1.20",
"version": "0.1.21",
"description": "Sitemap recursive loader for embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-loader-web": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@llm-tools/embedjs-loader-web": "0.1.21",
"debug": "^4.3.7",
"md5": "^2.3.0",
"sitemapper": "^3.2.18"
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-web/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "@llm-tools/embedjs-loader-web",
"version": "0.1.20",
"version": "0.1.21",
"description": "Web page loader for embedjs",
"dependencies": {
"@langchain/textsplitters": "^0.1.0",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-utils": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@llm-tools/embedjs-utils": "0.1.21",
"debug": "^4.3.7",
"html-to-text": "^9.0.5",
"md5": "^2.3.0"
Expand Down
4 changes: 2 additions & 2 deletions loaders/embedjs-loader-xml/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"name": "@llm-tools/embedjs-loader-xml",
"version": "0.1.20",
"version": "0.1.21",
"description": "XML loader for embedjs",
"dependencies": {
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"debug": "^4.3.7",
"fast-xml-parser": "^4.5.0",
"md5": "^2.3.0"
Expand Down
6 changes: 3 additions & 3 deletions loaders/embedjs-loader-youtube/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "@llm-tools/embedjs-loader-youtube",
"version": "0.1.20",
"version": "0.1.21",
"description": "Youtube transcript and channel recursive loader for embedjs",
"dependencies": {
"@langchain/textsplitters": "^0.1.0",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-utils": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"@llm-tools/embedjs-utils": "0.1.21",
"debug": "^4.3.7",
"md5": "^2.3.0",
"usetube": "^2.2.7",
Expand Down
6 changes: 3 additions & 3 deletions models/embedjs-anthropic/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"name": "@llm-tools/embedjs-anthropic",
"version": "0.1.20",
"version": "0.1.21",
"description": "Enable usage of Anthropic models with embedjs",
"dependencies": {
"@langchain/anthropic": "^0.3.8",
"@langchain/core": "^0.3.18",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@langchain/core": "^0.3.19",
"@llm-tools/embedjs-interfaces": "0.1.21",
"debug": "^4.3.7"
},
"type": "module",
Expand Down
4 changes: 2 additions & 2 deletions models/embedjs-cohere/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "@llm-tools/embedjs-cohere",
"version": "0.1.20",
"version": "0.1.21",
"description": "Enable usage of Cohere models with embedjs",
"dependencies": {
"@langchain/cohere": "^0.3.1",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@llm-tools/embedjs-interfaces": "0.1.21",
"cohere-ai": "^7.14.0"
},
"type": "module",
Expand Down
8 changes: 4 additions & 4 deletions models/embedjs-huggingface/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
{
"name": "@llm-tools/embedjs-huggingface",
"version": "0.1.20",
"version": "0.1.21",
"description": "Enable usage of HuggingFace models with embedjs",
"dependencies": {
"@huggingface/inference": "^2.8.1",
"@langchain/community": "^0.3.15",
"@langchain/core": "^0.3.18",
"@llm-tools/embedjs-interfaces": "0.1.20",
"@langchain/community": "^0.3.16",
"@langchain/core": "^0.3.19",
"@llm-tools/embedjs-interfaces": "0.1.21",
"debug": "^4.3.7"
},
"type": "module",
Expand Down
Loading

0 comments on commit 1bb273e

Please sign in to comment.