From 0a5f2cbd844d332437fc5426b5eb2191e49fdeb7 Mon Sep 17 00:00:00 2001 From: hammadb Date: Wed, 29 Jan 2025 08:13:59 -0800 Subject: [PATCH 1/7] [ENH] Rust Sqlite migrations --- Cargo.lock | 4 ++++ Cargo.toml | 3 ++- rust/sqlite/Cargo.toml | 6 ++++++ rust/sqlite/src/db.rs | 0 rust/sqlite/src/lib.rs | 1 + 5 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 rust/sqlite/Cargo.toml create mode 100644 rust/sqlite/src/db.rs create mode 100644 rust/sqlite/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a12da084c7c..42a85a1b988 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5674,6 +5674,10 @@ dependencies = [ "der", ] +[[package]] +name = "sqlite" +version = "0.1.0" + [[package]] name = "stable_deref_trait" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index 3e33e6369d7..4499b01ce51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] resolver = "2" -members = ["rust/benchmark", "rust/blockstore", "rust/cache", "rust/chroma", "rust/config", "rust/distance", "rust/error", "rust/frontend", "rust/garbage_collector", "rust/index", "rust/load", "rust/log", "rust/memberlist", "rust/storage", "rust/system", "rust/sysdb", "rust/types", "rust/worker", "rust/segment", "rust/python_bindings", "rust/mdac", "rust/tracing"] +members = ["rust/benchmark", "rust/blockstore", "rust/cache", "rust/chroma", "rust/config", "rust/distance", "rust/error", "rust/frontend", "rust/garbage_collector", "rust/index", "rust/load", "rust/log", "rust/memberlist", "rust/storage", "rust/system", "rust/sysdb", "rust/types", "rust/worker", "rust/segment", "rust/python_bindings", "rust/mdac", "rust/tracing", "rust/sqlite"] [workspace.dependencies] arrow = "52.2.0" @@ -52,6 +52,7 @@ chroma-system = { path = "rust/system" } chroma-sysdb = { path = "rust/sysdb" } chroma-tracing = { path = "rust/tracing" } chroma-types = { path = "rust/types" } +chroma-sqlite = { path = "rust/sqlite" } mdac = { path = "rust/mdac" } worker = { path = "rust/worker" } diff --git a/rust/sqlite/Cargo.toml b/rust/sqlite/Cargo.toml new file mode 100644 index 00000000000..247eabb6c48 --- /dev/null +++ b/rust/sqlite/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "sqlite" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/rust/sqlite/src/db.rs b/rust/sqlite/src/db.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/rust/sqlite/src/lib.rs b/rust/sqlite/src/lib.rs new file mode 100644 index 00000000000..b91801835d1 --- /dev/null +++ b/rust/sqlite/src/lib.rs @@ -0,0 +1 @@ +mod db; From 4b8bbeda17a570c63460de6856570c429ab57c0d Mon Sep 17 00:00:00 2001 From: hammadb Date: Wed, 29 Jan 2025 08:14:37 -0800 Subject: [PATCH 2/7] Sqlite db --- rust/sqlite/src/db.rs | 519 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 519 insertions(+) diff --git a/rust/sqlite/src/db.rs b/rust/sqlite/src/db.rs index e69de29bb2d..50399aec190 100644 --- a/rust/sqlite/src/db.rs +++ b/rust/sqlite/src/db.rs @@ -0,0 +1,519 @@ +use sha2::{Digest, Sha256}; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePool}; +use sqlx::{Executor, Row}; +use std::path::PathBuf; + +//////////////////////// SqliteDb //////////////////////////// + +#[derive(Clone)] +struct SqliteDBConfig { + // The SQLite database URL + url: String, + // TODO: change this to something bundled with binary + // The root directory where all migration files are stored + // The migration files are stored in subdirectories of this directory + migrations_root_dir: PathBuf, + hash_type: MigrationHash, + migration_mode: MigrationMode, +} + +/// Migration mode for the database +/// - Apply: Apply the migrations +/// - Validate: Validate the applied migrations and ensure none are unappliued +#[derive(Clone, PartialEq)] +enum MigrationMode { + Apply, + Validate, +} + +// TODO: +// - support memory mode, add concurrency tests +struct SqliteDb { + conn: SqlitePool, + config: SqliteDBConfig, + filename_regex: regex::Regex, +} + +impl SqliteDb { + pub async fn try_from_config(config: &SqliteDBConfig) -> Result { + // TODO: error type + + // PRAGMA legacy_alter_table=ON is required for some of our older + // migrations that use the ALTER TABLE ... RENAME TO ... syntax + // cur.execute("PRAGMA foreign_keys = ON") + // cur.execute("PRAGMA case_sensitive_like = ON") + // TODO: copy all other pragmas from python and add basic tests + let options = SqliteConnectOptions::new() + .filename(&config.url) + // Due to a bug in the python code, foreign_keys is turned off + // the python code enabled it in a transaction, however, + // https://www.sqlite.org/pragma.html states that foreign_keys + // is a no-op in a transaction. In order to be able to run our migrations + // we turn it off + .pragma("foreign_keys", "OFF") + .pragma("case_sensitive_like", "ON") + .create_if_missing(true); + let conn = SqlitePool::connect_with(options) + .await + .map_err(|e| e.to_string())?; + + // TODO: error type + let filename_regex = + regex::Regex::new(r"(\d+)-(.+)\.(.+)\.sql").map_err(|e| e.to_string())?; + + let db = Self { + conn, + config: config.clone(), + filename_regex, + }; + + db.validate_migrations_root_dir()?; + db.initialize_migrations_table().await?; + match config.migration_mode { + MigrationMode::Apply => { + let mut all_unapplied_migrations = Vec::new(); + for dir in migration_dirs.iter() { + let applied_migrations = db.get_existing_migrations(dir).await; + let source_migrations = db.get_source_migrations(dir).await?; + let unapplied = db.validate_migrations_and_get_unapplied( + applied_migrations, + source_migrations, + )?; + all_unapplied_migrations.extend(unapplied); + } + db.apply_migrations(all_unapplied_migrations).await?; + } + MigrationMode::Validate => { + // TODO: Test this + if !db.has_initialized_migrations().await { + return Err("Migrations table not initialized".to_string()); + } + for dir in migration_dirs.iter() { + let applied_migrations = db.get_existing_migrations(dir).await; + let source_migrations = db.get_source_migrations(dir).await?; + let unapplied = db.validate_migrations_and_get_unapplied( + applied_migrations, + source_migrations, + )?; + if !unapplied.is_empty() { + return Err("Unapplied migrations found".to_string()); + } + } + } + } + Ok(db) + } + + //////////////////////// Migrations //////////////////////// + + // TODO: Real error + /// Apply all migrations in a transaction + /// Arguments: + /// - migrations: Vec - The migrations to apply + async fn apply_migrations(&self, migrations: Vec) -> Result<(), String> { + let mut tx = self.conn.begin().await.map_err(|e| e.to_string())?; + for migration in migrations { + println!("Applying migration: {}", migration.filename); + // Apply the migration + tx.execute("PRAGMA foreign_keys = ON") + .await + .map_err(|e| e.to_string())?; + tx.execute(sqlx::query(&migration.sql)) + .await + .map_err(|e| e.to_string())?; + println!("Applied migration: {}", migration.filename); + + // Bookkeeping + let query = r#" + INSERT INTO migrations (dir, version, filename, sql, hash) + VALUES ($1, $2, $3, $4, $5) + "#; + let query = sqlx::query(query) + .bind(&migration.dir) + .bind(migration.version) + .bind(&migration.filename) + .bind(&migration.sql) + .bind(&migration.hash); + tx.execute(query).await.map_err(|e| e.to_string())?; + } + tx.commit().await.map_err(|e| e.to_string())?; + Ok(()) + } + + /// Validate the migrations root directory + fn validate_migrations_root_dir(&self) -> Result<(), String> { + // TODO: replace with ChromaError + for dir in migration_dirs.iter() { + let path = self.config.migrations_root_dir.join(dir.as_str()); + if !path.exists() { + return Err(format!("Migration directory {:?} does not exist", path)); + } + } + Ok(()) + } + + // TODO: Real error + /// Validate migration sequence and get the migrations that need to be applied + /// Arguments: + /// - applied_migrations: Vec - The migrations that have been applied, in ascending version order + /// - source_migrations: Vec - The migrations that are on disk, in ascending version order + /// Returns: + /// - Vec - The migrations that need to be applied + fn validate_migrations_and_get_unapplied( + &self, + applied_migrations: Vec, + source_migrations: Vec, + ) -> Result, String> { + for (db_migration, source_migration) in + applied_migrations.iter().zip(source_migrations.iter()) + { + if db_migration.version != source_migration.version { + return Err(format!( + "Inconsistent version: db={}, source={}", + db_migration.version, source_migration.version + )); + } + if db_migration.hash != source_migration.hash { + return Err(format!( + "Inconsistent hash: db={}, source={}", + db_migration.hash, source_migration.hash + )); + } + } + + let unapplied = source_migrations[applied_migrations.len()..].to_vec(); + Ok(unapplied) + } + + /// Initialize the migrations table + /// Note: + /// - This function is idempotent + async fn initialize_migrations_table(&self) -> Result<(), String> { + let query = r#" + CREATE TABLE IF NOT EXISTS migrations ( + dir TEXT NOT NULL, + version INTEGER NOT NULL, + filename TEXT NOT NULL, + sql TEXT NOT NULL, + hash TEXT NOT NULL, + PRIMARY KEY (dir, version) + ) + "#; + sqlx::query(query) + .execute(&self.conn) + .await + .map_err(|e| e.to_string())?; + Ok(()) + } + + /// Check if the migrations table has been initialized + /// Returns: + /// - bool - True if the migrations table has been initialized + async fn has_initialized_migrations(&self) -> bool { + let query = r#" + SELECT name FROM sqlite_master WHERE type='table' AND name='migrations' + "#; + let row = sqlx::query(query) + .fetch_one(&self.conn) + .await + .expect("Expect it to be fetched"); + let name: String = row.get("name"); + name == "migrations" // Sanity check + } + + /// Get existing migrations for a given directory + /// Arguments: + /// - dir_name: str - The name of the directory that contains the migrations + /// Returns: + /// - Vec - A list of migrations + /// Notes + /// - dir_name has to be held constant for a given migration directory + /// - The migrations are sorted by version in ascending order + /// - The dir_name is consistent with the python implementation + async fn get_existing_migrations(&self, dir: &MigrationDir) -> Vec { + let query = r#" + SELECT dir, version, filename, sql, hash + FROM migrations + WHERE dir = $1 + ORDER BY version ASC + "#; + let rows = sqlx::query(query) + .bind(dir.as_str()) + .fetch_all(&self.conn) + .await + .expect("Expect it to be fetched"); + + let mut migrations = Vec::new(); + for row in rows { + let dir: String = row.get("dir"); + let version: i32 = row.get("version"); + let filename: String = row.get("filename"); + let sql: String = row.get("sql"); + let hash: String = row.get("hash"); + migrations.push(Migration { + dir, + version, + filename, + sql, + hash, + }); + } + migrations + } + + // TODO: REAL ERROR + /// Get the migrations that are on disk + /// Arguments: + /// - dir: str - The name of the directory that contains the migrations + /// Returns: + /// - Vec - A list of migrations found on disk, sorted by version in ascending order + /// Notes: + /// - Uses the migrations_root_dir of this SqlDB instance + async fn get_source_migrations(&self, dir: &MigrationDir) -> Result, String> { + let on_disk_path = self.config.migrations_root_dir.join(dir.as_str()); + let mut migrations = Vec::new(); + let mut read_dir = tokio::fs::read_dir(on_disk_path) + .await + .map_err(|e| e.to_string())?; + + while let Some(entry) = read_dir.next_entry().await.map_err(|e| e.to_string())? { + let path = entry.path(); + let filename = match path.file_name() { + Some(filename) => filename, + None => return Err("Filename is None".to_string()), + }; + let filename = match filename.to_str() { + Some(filename) => filename, + None => return Err("Filename is not valid".to_string()), + }; + let (version, _) = self.parse_migration_filename(filename)?; + let sql = tokio::fs::read_to_string(&path) + .await + .map_err(|e| e.to_string())?; + let hash = match self.config.hash_type { + MigrationHash::SHA256 => { + let mut hasher = Sha256::new(); + hasher.update(sql.as_bytes()); + format!("{:x}", hasher.finalize()) + } + MigrationHash::MD5 => { + let hash = md5::compute(sql.as_bytes()); + format!("{:x}", hash) + } + }; + migrations.push(Migration { + dir: dir.as_str().to_string(), + version, + filename: filename.to_string(), + sql, + hash, + }); + } + // TODO: Make a Vec wrapper type that enforces sorting + migrations.sort_by(|a, b| a.version.cmp(&b.version)); + Ok(migrations) + } + + // Parse the migration filename + // Arguments: + // - filename: str - The filename of the migration + // Returns: + // - (i32, str) - The version and scope of the migration + // Notes + // - Format is -..sql + // - e.g, 00001-users.sqlite.sql + // - scope is unused, it is legacy from the python implementation. It is + // written but never read + fn parse_migration_filename(&self, filename: &str) -> Result<(i32, String), String> { + let regex_match = self.filename_regex.captures(filename); + let groups = match regex_match { + Some(groups) => groups, + // TODO: Error + None => return Err(format!("Invalid migration filename: {}", filename)), + }; + + // Parse version + let version = match groups.get(1) { + Some(version) => version, + None => return Err("Failed to find version".to_string()), + }; + let version = match version.as_str().parse::() { + Ok(version) => version, + Err(e) => return Err(e.to_string()), + }; + + // Parse scope + let scope = match groups.get(3) { + Some(scope) => scope, + None => return Err("Failed to find scope".to_string()), + }; + let scope = scope.as_str().to_string(); + + Ok((version, scope)) + } +} + +#[derive(Clone)] +struct Migration { + dir: String, + filename: String, + version: i32, + sql: String, + hash: String, +} + +enum MigrationDir { + SysDb, + MetaDb, + EmbeddingsQueue, +} + +const migration_dirs: [MigrationDir; 3] = [ + MigrationDir::SysDb, + MigrationDir::MetaDb, + MigrationDir::EmbeddingsQueue, +]; + +impl MigrationDir { + fn as_str(&self) -> &str { + match self { + Self::SysDb => "sysdb", + Self::MetaDb => "metadb", + Self::EmbeddingsQueue => "embeddings_queue", + } + } +} + +#[derive(Clone)] +enum MigrationHash { + SHA256, + MD5, +} + +//////////////////////// SqliteSysDb //////////////////////// + +struct SqliteSysDb { + conn: SqlitePool, +} + +impl SqliteSysDb { + pub fn new(conn: SqlitePool) -> Self { + Self { conn } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sqlx::Row; + use tempfile::tempdir; + + //////////////////////// Test Helpers //////////////////////// + + fn test_migration_dir() -> PathBuf { + let migration_dir = "/Users/hammad/Documents/chroma/chromadb/migrations"; + PathBuf::from(migration_dir) + } + + fn existing_test_db_path() -> String { + // TODO: return bundled path + "/Users/hammad/Documents/chroma/chroma/chroma.sqlite3".to_string() + } + + fn new_test_db_path() -> String { + // TODO: Make tmpfile work + // let dir = tempdir().expect("Expect it to be created"); + // let path = dir.path().join("chroma.sqlite3"); + let path = "/Users/hammad/Documents/chroma/chroma/chromaTEST.sqlite3".to_string(); + // remove the file if it exists + std::fs::remove_file(&path).unwrap_or_default(); + path + } + + //////////////////////// SqliteDb //////////////////////// + + #[tokio::test] + async fn test_sqlite_db() { + let config = SqliteDBConfig { + url: "sqlite::memory:".to_string(), + migrations_root_dir: test_migration_dir(), + hash_type: MigrationHash::MD5, + migration_mode: MigrationMode::Apply, + }; + let db = SqliteDb::try_from_config(&config) + .await + .expect("Expect it to be created"); + + // Check if migrations table exists + let query = r#" + SELECT name FROM sqlite_master WHERE type='table' AND name='migrations' + "#; + let row = sqlx::query(query) + .fetch_one(&db.conn) + .await + .expect("Expect it to be fetched"); + let name: String = row.get("name"); + assert_eq!(name, "migrations"); + } + + #[tokio::test] + async fn test_migrations_validate_on_existing_db() { + let config: SqliteDBConfig = SqliteDBConfig { + url: existing_test_db_path(), + migrations_root_dir: test_migration_dir(), + hash_type: MigrationHash::MD5, + migration_mode: MigrationMode::Validate, + }; + let db = SqliteDb::try_from_config(&config) + .await + .expect("Expect it to be created"); + + // Check if migrations table exists + let query = r#" + SELECT name FROM sqlite_master WHERE type='table' AND name='migrations' + "#; + let row = sqlx::query(query) + .fetch_one(&db.conn) + .await + .expect("Expect it to be fetched"); + let name: String = row.get("name"); + assert_eq!(name, "migrations"); + } + + #[tokio::test] + async fn test_migrations_get_applied_on_new_db() { + let config = SqliteDBConfig { + url: new_test_db_path(), + migrations_root_dir: test_migration_dir(), + hash_type: MigrationHash::MD5, + migration_mode: MigrationMode::Apply, + }; + let db = SqliteDb::try_from_config(&config) + .await + .expect("Expect it to be created"); + for dir in migration_dirs { + let migrations = db.get_existing_migrations(&dir).await; + let on_disk_path = test_migration_dir().join(dir.as_str()); + // See how many files are in the directory + let files = std::fs::read_dir(on_disk_path).unwrap(); + let num_files = files.count(); + assert_eq!(migrations.len(), num_files); + } + } + + // TODO: more tests + // - add test migrations + // - tamper with one and test + // - add new migration and test + // - reorder migrations + + //////////////////////// SqliteSysDb //////////////////////// + + #[tokio::test] + async fn test_sqlite_sysdb() { + let conn = SqlitePool::connect("sqlite::memory:") + .await + .expect("Expect it to be connected"); + let sysdb = SqliteSysDb::new(conn); + } +} From 77aa506c9e644136eebf2d57ac99953f2683f74f Mon Sep 17 00:00:00 2001 From: hammadb Date: Wed, 29 Jan 2025 08:23:23 -0800 Subject: [PATCH 3/7] cln --- Cargo.lock | 459 +++++++++++++++++++++++++++++++++++- Cargo.toml | 4 + rust/sqlite/Cargo.toml | 8 + rust/sqlite/sqlite::memory: | Bin 0 -> 159744 bytes rust/sqlite/src/config.rs | 31 +++ rust/sqlite/src/db.rs | 31 +-- rust/sqlite/src/lib.rs | 1 + 7 files changed, 496 insertions(+), 38 deletions(-) create mode 100644 rust/sqlite/sqlite::memory: create mode 100644 rust/sqlite/src/config.rs diff --git a/Cargo.lock b/Cargo.lock index 42a85a1b988..75ffc5e65d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1016,6 +1016,9 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "bitpacking" @@ -1686,6 +1689,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -1770,6 +1788,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -1925,6 +1952,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -1964,6 +2002,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] @@ -2000,6 +2039,12 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "downcast-rs" version = "1.2.1" @@ -2018,10 +2063,10 @@ version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" dependencies = [ - "der", + "der 0.6.1", "elliptic-curve", "rfc6979", - "signature", + "signature 1.6.4", ] [[package]] @@ -2029,6 +2074,9 @@ name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +dependencies = [ + "serde", +] [[package]] name = "elliptic-curve" @@ -2038,12 +2086,12 @@ checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" dependencies = [ "base16ct", "crypto-bigint 0.4.9", - "der", + "der 0.6.1", "digest", "ff", "generic-array", "group", - "pkcs8", + "pkcs8 0.9.0", "rand_core", "sec1", "subtle", @@ -2081,6 +2129,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "event-listener" version = "5.3.1" @@ -2465,6 +2524,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -2716,6 +2786,15 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "heck" version = "0.4.1" @@ -2746,6 +2825,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -3450,6 +3538,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "levenshtein_automata" @@ -3543,6 +3634,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -3709,6 +3811,12 @@ dependencies = [ "digest", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "mdac" version = "0.1.0" @@ -3925,6 +4033,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -4318,6 +4443,15 @@ dependencies = [ "serde", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -4411,14 +4545,35 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der 0.7.9", + "pkcs8 0.10.2", + "spki 0.7.3", +] + [[package]] name = "pkcs8" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" dependencies = [ - "der", - "spki", + "der 0.6.1", + "spki 0.6.0", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.9", + "spki 0.7.3", ] [[package]] @@ -5067,6 +5222,26 @@ dependencies = [ "byteorder", ] +[[package]] +name = "rsa" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c75d7c5c6b673e58bf54d8544a9f432e3a925b0e80f7cd3602ab5c50c55519" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8 0.10.2", + "rand_core", + "signature 2.2.0", + "spki 0.7.3", + "subtle", + "zeroize", +] + [[package]] name = "rtrb" version = "0.3.1" @@ -5338,9 +5513,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" dependencies = [ "base16ct", - "der", + "der 0.6.1", "generic-array", - "pkcs8", + "pkcs8 0.9.0", "subtle", "zeroize", ] @@ -5594,6 +5769,16 @@ dependencies = [ "rand_core", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "siphasher" version = "1.0.1" @@ -5623,6 +5808,9 @@ name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +dependencies = [ + "serde", +] [[package]] name = "snafu" @@ -5671,12 +5859,217 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" dependencies = [ "base64ct", - "der", + "der 0.6.1", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der 0.7.9", ] [[package]] name = "sqlite" version = "0.1.0" +dependencies = [ + "md5", + "regex", + "sha2", + "sqlx", + "tempfile", + "tokio", +] + +[[package]] +name = "sqlx" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4410e73b3c0d8442c5f99b425d7a435b5ee0ae4167b3196771dd3f7a01be745f" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a007b6936676aa9ab40207cde35daab0a04b823be8ae004368c0793b96a61e0" +dependencies = [ + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.2", + "hashlink", + "indexmap 2.6.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.4", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3112e2ad78643fef903618d78cf0aec1cb3134b019730edb039b69eaf531f310" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.89", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e9f90acc5ab146a99bf5061a7eb4976b573f560bc898ef3bf8435448dd5e7ad" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.89", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.6.0", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.4", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.6.0", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.4", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f85ca71d3a5b24e64e1d08dd8fe36c6c95c339a896cc33068148906784620540" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "tracing", + "url", +] [[package]] name = "stable_deref_trait" @@ -5690,6 +6083,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.10.0" @@ -6517,12 +6921,33 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + [[package]] name = "unicode-width" version = "0.1.14" @@ -6675,6 +7100,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -6792,6 +7223,16 @@ dependencies = [ "rustix", ] +[[package]] +name = "whoami" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +dependencies = [ + "redox_syscall", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 4499b01ce51..a0fac226436 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,10 @@ tracing-bunyan-formatter = "0.3" tracing-opentelemetry = "0.28.0" tracing-subscriber = { version = "0.3", features = ["env-filter"] } uuid = { version = "1.11.0", features = ["v4", "fast-rng", "macro-diagnostics", "serde"] } +sqlx = { version = "0.8.3", features = [ "runtime-tokio", "sqlite"] } +sha2 = "0.10.8" +md5 = "0.7.0" +regex = "1.11.1" chroma-benchmark = { path = "rust/benchmark" } chroma-blockstore = { path = "rust/blockstore" } diff --git a/rust/sqlite/Cargo.toml b/rust/sqlite/Cargo.toml index 247eabb6c48..d6e9949bca2 100644 --- a/rust/sqlite/Cargo.toml +++ b/rust/sqlite/Cargo.toml @@ -4,3 +4,11 @@ version = "0.1.0" edition = "2021" [dependencies] +sqlx = { workspace = true } +sha2 = { workspace = true} +regex = { workspace = true } +tokio = { workspace = true} +md5 = { workspace = true} + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/rust/sqlite/sqlite::memory: b/rust/sqlite/sqlite::memory: new file mode 100644 index 0000000000000000000000000000000000000000..e249e220ce1f305536affb5d4d624f468c1f355a GIT binary patch literal 159744 zcmeI5O>i4Wc7QPi36P*9$d+YTqO@vB>!n~3GzR}D#&KX0l2PW51cH{GIE5O_G)S!Q zLt%!LxK(=qDLcs?cdN3MOAa}1RdPvHYAdJI9&*^j9#W~Ty=-bPd&wbNsd6O;m$P|2 zJwHA8BfXY)Dfyu!anRk<^ZI+;uU~h+p*HWYHVjJYoo-7pNYZ!SCrQ3PA;jnNO~BuI z_-lXOfQw1{3;dLX>rs~zzQynCPO-w_&sZ4}{_O0HGyd71%xbggna^i_I`hNn+Vt(< zr@@bcR|85QK6QEW*Z$x5f8`NBEd*LFZI?&3DeM?45=Sd=Z+Nu3eDfr>2^a~AO`KW z7Rc^)yR#q)B4$UUqsds&pY?@*yLxf|=myeZ>l^V^b z*6gSc$vdkX??jDn1S7IM-MctKj6n&kXTM)6SGEePmYN2Ok*&?r`dv~t^fc&cf#^n; z^@Dw-xksbPtAWVf>yoe0*60)cQ4>Z&RoOE-=J#q#c~aHsqiRE|%I@{0{VT!9_3KhE zYl`Y-xL2=vxVY)$R*1j{wAF&H2lbLl@q>z}ss*b6d#DM9tZ!7v`qt`dD7rish&+%- z3G24JMGb{H}@xHF(dfp4JTJTdfiXLAiLRST3$F7dMGhTnGA5 z-n@tOfH&N3o7HMTGiE+dZU8A$TlN^^$@OMOu`^(xuJTl2_z_dx@T_Pkn!Gel&#sID zGCg#7;WbmyM~}RybhrG>a5VZL521Q5Lv!D zsux-}>Q&jky4cIV8jM`MDjgP0jCQ*HGvGmZ!%n{8A{;(OjDuY|v=^zJ;TdYD*`%t` z=(PE5LML^zSSNL8g`#g=3`FkV9@W^E0)3OVm9|Q&&CWJ65}t&q-o;mfk=wVW9*jnq zh5EAlzD{~T^ktef_z3Ga7e=E+bwg?Gvhm@_&h!Z&x{BLiy9b4$=?j5K>E@_rxis`@ zLiqAZ`sI_}CJVP+I83}0jNH5_J-uyG5VC~Jmp!e&A>@qf2_zOHd+o-fJxaElotn~Y z9+(ESt+e_(hfox-5N052icZbq8X#=ztY=u$I3EpmnSn7#guY`JGVL4LScf6DS_HRY zxv;rhSSf-7&}r8j+tqLDopu!_AHlNOqm4r!IufSpvpae`57kR3%Qop$p8{-KUO$m52w17)@w`?PIV5Q@gm1|o&I zQQEOYtIF2JxsT2SBXe_7FKkk_Qmr3nJ%p{qF?>BOGcEGlvw_HFY?LsTL9N>JyJMN1 zlrx7{LcvHZCLKLA*;{IGqbUWGhal|2G<&D+dXdl6H2H32?hv`Wj1XAmC zn!{WPVQ%(LecX8{1_O}~lcV^$mdXKI70jQLIDHrk1S83$^i(n_54mqhQqaSCNC6AC zkYf`UY?PaOI+?rj!H{1dmd`dZ&Sz^iw)xx!u7-Q}Z-hS!{A%j6$qW7m6JcMoe{KEczaNpAU__!H&#mbPx>lQs!zUlJ z%t6fU1+jL=h_zv*zR1^P><8HF%1%QkE#)EA?Io1H=-x!yowj)sbF~&j9xdAzrwZ%+ z-Q$ew@5QrR3WZARo5ga4xf~l#d$@p`;6f}{$Q$HNd1KAVTMDg|H||*rUAsj~p~7kf z_8s^xg`LW)0S~{h1{)Y|^-H06I+aYP6}G3g>wutj#;sbbGOh-a?|K^p>qp; znwQ$M+<2{Fu3waHgX)VO#k;rYq_DC=mN!n4r%ey6!ltv=G;hQe+7*yC5HbRF)ykq#KVMN%@%9t79qVWZi!9&A?6s#Q#{fGTZb4twCZ*J>|`Zh@$+LH}gv=sxkiDN2=W zDwE46)3tgct)(*YloGE~Es@M*WI5BTO@W^89=^r(JjL{!j&*4Z$ixb-A>C)U3Qk(pi5B6Cf`P|MzS6n*=0?`X7yR~!9yFalOg45ax#-ubBRnX zSFb7QdOlN6WNWmpsj`+zXo>I5O|pUb1FJJnvPPz2mY*H7&6O^+)H8FG?wC`0@q^N4 zWz$+G2-`cu_a^Lxhv}Y!r5{tHBfywh>1(m18jne^Z&hRL6#l!zvQ=UFM`kY^=3{-8 zdj+1g-LPE&fCM&c+1%nI8m%i}EbYZB8*ok&EZ<_I4C}6YtA*tvcQc&jCEMnDzp%Om zJG8gvt%|^DV5{Ky=n`YdEMVUgVETl~xW72*fKBZ3nSE=kw4&RNP5;Af;b8bMFQ2!M zKa=fwxbmiUHamz&; zch9^n>)3X1A5Rd*j^pFrlfK?Fpo4>S3#LH6;CA9tSrZADwRy8 z^Kvif4@9H~2Yd$bGl`SpOz$8N?BfVA#F@;w&E8O33Nc%p(BkQAN=t%emh)Nh`HSuryW^j4GPqQ^Q5d?qX1m9s zXul=1WOS$P;io{SV28v+C$y_p>(6WiNqRFb9iDy%FMNK8*wL}6I^w(!X@NYPr@rBpIneg|*-wp4E)$sp> z|1SRok zoFGlT3K3{~bRv9lGBz`ieD2KTB}p>R%4(Bmg)jay!k3BJ*~v>YRTlkTqb(7nVc|ns2_OL^fCP{L5;D%*x!4FKfCP{L560301`j~NB{{S z0VIF~kN^@u0!RP}oJs60301`j~NB{{S0VIF~kN^@u0!RP}oJs%%8&pOKmter2_OL^fCP{L5b(KH4u>f zy$XM?oJwI)C?tRckN^@u0!RP}AOR$R1dsp{KmthMBm_bszwfeD&@cIYXYl<0B-pSF z5Z4r z`;%F1Ha+wC%ui>2I9;2*9sD%-QSfR&3B;!^PyX8f8~=~|iHXlA{zCe#r22mDdzZJr zH+v%(DHNpMbwjB&DQ(rLrZw8z)%squY0xJ|RqLpF<2z$=t}d60g-VfB3h%5I$^7Ul z=5IALQd+MR?-tACUb(bZD1S)aD}J~@^zV>&RyW>>_CnXWG}lH+qjuT`g+GIGS~*va zmBuPKS{gOJKvX%}yF`MKd|v9UjS{AzJsBbhq|O~H1QeRTrPJ+(wm=NpZ7q=9?RIBD z#3E)#qvS!U^6th~g_JiQlva9E*MgCyrEzLg3}uLL8YEvnRytN{G@n|tqdw#cGrkdw z$ntdW;s`{85?IfEzf`Vl6;`cIGgyplZI;&WlDeU%q1QkUF}kc*?kmkb8ckjeMDAXf ze2un7pXiU84TDydJ)>iOueOvYRh>SnHnghjUSHb35{z8GF7>jesBVUP^_qu^n@(

6aO3>FA|C(Dq$}_q^(9&0xOODq%>Li+76U;`(xNlQ_k7pdaPUdq@v>!|k?Ntrj$6 z=JVtRkTSJpk9eM3Z*~+r1I)%MPZf*-G1U#viiV=eOXKwH$|xYyLx&e$GZlUG$csvM z%g+o)qYokx1hZP(*9p8VoJTmF#GB8%rMHLhdetsf460QXgH%du#m!1#?Vi(#$V%}} zVQaNQmbc2Fpi0##!f)PMFWuiN5@s;09=hOa5JUl#Z$<)<<*TE5p>?BPmF=sGz5J`e z$knUTVbR2Br`taR9)vgS~>0sv3<>o8KmMQa6irQioP3 z`qsrj@P$-(d5Qvm+j%t=mL$4-;FR!FuKIv_;aNC8$ z#7n`*&70EG+a?7eOSpX5)A}1i&bXdHVllGUZamtfWV_j^Db41AX+YabtG{yyMF9(8 z2C}B;)GV$6!nV$OhUHZ7(O{Pu7=uLUJ9Z({zLAY}7-FkMaAKAVo6CijA~*n@cD=D( z{kGm|S7GuIESo*rn6wLyM0}XZ=L3=AwNWZ^9$HN}moLfTlieaax12vrp9@B=U6Y>X zP1QIV&PCXx7AJYk;1oE2L6w+8$uJedFUNA{MMhCKYq9w{wJi{aI?6;Ww)MR(CB_ce zNd+7y7{vzJQFPL2HxJ++>ga8&$UFilWiYI95PnkSlx~G5iDnb%?f;&p434gwN9rw%#{%4X7AL;orhvD z5cx1Uimz*_9H3Rf{5gr!hp|8~l1xfZC6n@y`-UV1J*-U|7rG*XJ^iA1#V7tC;xp?_J2QAo%#Fe|DC8#f8_tZw>JES1V$1#jK|Eq z<);_S4LRprQ?J6I&7KF8Z}z^cxA;CBId;0{K`psjgI%R9%vQ#)=34l z%8!mGaiV|b=DCQkA0wg_!7~;i9LX53pH##Xgak!r5^{bHvgJ*b+?=VZj%N=hb>{!6p7I|>lU9UuzvF|rv-ehlZ znWIH-A!|n9kr4yofP6L>a{dlRHn|XtyeCV&$7ZaTQ@wLu$Ij}VK(WDWJ65r@zEb=^ zh>@{-o#%EI7BE-^xdjJZeF16Y!MhMS7>OYDqft@|VNrumv_l$4 z3N73{R%W3b%nqi=LXt<~`8R@*B1lX}N$eco4$5msKaP~wxp%CzPAOYfCqb0->lX_M zPTQY8+gQLrG+=syy-VK=f+?ErEqe!kzmMRm+=&wA)0;I-92eXP;hD@65@S5YVG9wl zorkmyLC0^+!)S&RXsm~@a z_#aG!A?cZaj>t?fBGHfM*7O5itIfpWlaE>9KruH8B-ZX2u{OLnu*hSw*$;@>s_Zm$ z(o!B$-43DF7u}mkyVEvrVxg9cA&;YJM;uil-hbEi8UBEgrAwhuX??R;uCS294W~U^ zz)c7tT&|Ee$er@Wnv=H_S}AYbvjW}i7A=Jes}*<)g1<9ir}Anb!nd#nuT!|yFNNai zR5G1bXgyCe$!t7PtEW>`Nx(Nbok`Z^!{RikXzl2`Tt(ANMY)&|jT(bUSFc*iRPA&% zojl%w_&7U65cCAIEDw%#15tINK+qk;U)DR_7S$F*pw)swSO_cW)QP!phcHaA+f3Ew z+(F>NjLF;6sRs6(bj`<+eMQ~dYYA2Nsa$wI3(hr_&fP@M@)lgg-ywh&I4W_PrTL^R z%Sv3%=JJZB)ibGlMpiR&BCGcDLEw15r*Muz#xW~)aqdccNp8Yc0d#JmPxDfHRv53= zFyoh%ZiDKJ9>u%2=%lc+LY6mHx7OB&#-1*PWHk%Quho=fDx;(likz!ulv=hftNC0W zUfVgM5UD27PhRJ;2bk;`QFeR8+RkoYEAFqtdlW?LKvQ5K6Mb)Yx6_3%OY?moqV7<5 z>4OJV!izy&uI2>-v6l)cSKn(I@P(IEcxl#68twf~bDvs$1olr9(x#8CKBBZWXzT6) z+gCAmcir5wNxgaQsQ;0C6bwpEN6PPDbVxX!?(DerzXQSsf3pJ-rOV`h(EA8^CW9zDrUvm#%yz?3oZ4`9Hl$v;;Z;UX|u9v z#R&_qcMP9F?7(9)JQpnem>L}c#>`4zi;(K^m<0P)HO5ZizdI~j73QwN$Zg)Q@?8P* z$#&-%_ErEu0-LpLzT!hPT35hW+QI8=z&TB@e2a}T#9iN8Ei4zgo8g3)v)5eSFRX6C z8(O#Kt%|^DU{Ub%(Iv)^S-`$0!1M`|aer~r0h`$6GyB$7X+^gkoBoH}!ol!in^wGi z{F&^X2Ui|8MA;`c!Bg1|zr0?Yn7K&@R zXyfjgw`CpM4({U#!g$;~PrB&o2^r>qaD6=?2i%lYE(hk3=ISaalcwT{q?(oGI8CLJ z$#h=s1^t1D^x%Nc0DdNMQk>}>1cH4WA%-}UIk(v_)GURVElz0hbT*|W!7|HvHNjk{ zdM1%p^GQWHx;p`O`t47`T$BkWO2S?nj5N=-!#l=$^5OhN-xhPnKR>o{sq(X*kt@bf zwBM3hGP=|D@V7uu!5b1LDo^0JExWtRycA00<)jK8V>+IU!%`xjW;#rza`9xA=JIml z=r+XQ&lEr24cU|q!mq8Cb!btvSa*)*-Lu1iHyFGiV5$3HD00;F5Y4 z*luwiV`LpOyDp*~*dus4wv#vbsy1^LU}ph*=X$-Cq-oeD({eS~UP$NCa$LzJXfA#D z;9M{^v-~tN&;suKxDuUIW=mM>*F20C_{;pVhd+Fz(^5!FX47e{4m(nb8qLvMGA`Fs zTDGnNYbBM-9F@-oFG(ApTv#*r8f(_}jlJ<0gG1$p3Y($sIf2y`_>r-9_r_6!U_b_T!d9^?X**z%@@PR8ewu zSS#1#aLy2~Wm35sH2Djxy zazaUGFA)^gYA z!@^SCw%O}fZ-D9_?l%ebFT!0(9WC&k5`JuFMt2EoZ>PAq^yU-z=-(RQgmMXSw~kr2 zcd2+KqH#U;H%s+5;#vK&FSp*~Or9A0GGLxyx2Bd->*<7=s?`#iY=$OeEtAW{l~gL8 zujkSkw*H?8|Hc>o4F16n5A(jsR)!O9$18csO-2eap literal 0 HcmV?d00001 diff --git a/rust/sqlite/src/config.rs b/rust/sqlite/src/config.rs new file mode 100644 index 00000000000..446c01bd0c6 --- /dev/null +++ b/rust/sqlite/src/config.rs @@ -0,0 +1,31 @@ +use std::path::PathBuf; + +#[derive(Clone)] +pub(crate) struct SqliteDBConfig { + // The SQLite database URL + pub(crate) url: String, + // TODO: change this to something bundled with binary + // The root directory where all migration files are stored + // The migration files are stored in subdirectories of this directory + pub(crate) migrations_root_dir: PathBuf, + pub(crate) hash_type: MigrationHash, + pub(crate) migration_mode: MigrationMode, +} + +/// Migration mode for the database +/// - Apply: Apply the migrations +/// - Validate: Validate the applied migrations and ensure none are unappliued +#[derive(Clone, PartialEq)] +pub(crate) enum MigrationMode { + Apply, + Validate, +} + +/// The hash function to use for the migration files +/// - SHA256: Use SHA256 hash +/// - MD5: Use MD5 hash +#[derive(Clone)] +pub(crate) enum MigrationHash { + SHA256, + MD5, +} diff --git a/rust/sqlite/src/db.rs b/rust/sqlite/src/db.rs index 50399aec190..522a9c09fcc 100644 --- a/rust/sqlite/src/db.rs +++ b/rust/sqlite/src/db.rs @@ -1,31 +1,9 @@ +use crate::config::{MigrationHash, MigrationMode, SqliteDBConfig}; use sha2::{Digest, Sha256}; use sqlx::sqlite::{SqliteConnectOptions, SqlitePool}; use sqlx::{Executor, Row}; use std::path::PathBuf; -//////////////////////// SqliteDb //////////////////////////// - -#[derive(Clone)] -struct SqliteDBConfig { - // The SQLite database URL - url: String, - // TODO: change this to something bundled with binary - // The root directory where all migration files are stored - // The migration files are stored in subdirectories of this directory - migrations_root_dir: PathBuf, - hash_type: MigrationHash, - migration_mode: MigrationMode, -} - -/// Migration mode for the database -/// - Apply: Apply the migrations -/// - Validate: Validate the applied migrations and ensure none are unappliued -#[derive(Clone, PartialEq)] -enum MigrationMode { - Apply, - Validate, -} - // TODO: // - support memory mode, add concurrency tests struct SqliteDb { @@ -384,12 +362,6 @@ impl MigrationDir { } } -#[derive(Clone)] -enum MigrationHash { - SHA256, - MD5, -} - //////////////////////// SqliteSysDb //////////////////////// struct SqliteSysDb { @@ -405,6 +377,7 @@ impl SqliteSysDb { #[cfg(test)] mod tests { use super::*; + use crate::config::MigrationHash; use sqlx::Row; use tempfile::tempdir; diff --git a/rust/sqlite/src/lib.rs b/rust/sqlite/src/lib.rs index b91801835d1..f1377225c73 100644 --- a/rust/sqlite/src/lib.rs +++ b/rust/sqlite/src/lib.rs @@ -1 +1,2 @@ +mod config; mod db; From 7e428f5dfd1d916a19560e6b9f1103f5e61548ed Mon Sep 17 00:00:00 2001 From: hammadb Date: Thu, 30 Jan 2025 14:51:39 -0800 Subject: [PATCH 4/7] context save --- Cargo.lock | 35 +++++++++++++++++++++++++++++ rust/sqlite/Cargo.toml | 1 + rust/sqlite/sqlite::memory: | Bin 159744 -> 0 bytes rust/sqlite/src/db.rs | 8 +------ rust/sqlite/src/lib.rs | 1 + rust/sqlite/src/migration_files.rs | 16 +++++++++++++ 6 files changed, 54 insertions(+), 7 deletions(-) delete mode 100644 rust/sqlite/sqlite::memory: create mode 100644 rust/sqlite/src/migration_files.rs diff --git a/Cargo.lock b/Cargo.lock index 91d071085b6..8f0905a3d7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5250,6 +5250,40 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3f94e84c073f3b85d4012b44722fa8842b9986d741590d4f2636ad0a5b14143" +[[package]] +name = "rust-embed" +version = "8.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa66af4a4fdd5e7ebc276f115e895611a34739a9c1c01028383d612d550953c0" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6125dbc8867951125eec87294137f4e9c2c96566e61bf72c45095a7c77761478" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.89", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5347777e9aacb56039b0e1f28785929a8a3b709e87482e7442c72e7c12529d" +dependencies = [ + "sha2", + "walkdir", +] + [[package]] name = "rust-stemmers" version = "1.2.0" @@ -5880,6 +5914,7 @@ version = "0.1.0" dependencies = [ "md5", "regex", + "rust-embed", "sha2", "sqlx", "tempfile", diff --git a/rust/sqlite/Cargo.toml b/rust/sqlite/Cargo.toml index d6e9949bca2..e6ba7944e3f 100644 --- a/rust/sqlite/Cargo.toml +++ b/rust/sqlite/Cargo.toml @@ -9,6 +9,7 @@ sha2 = { workspace = true} regex = { workspace = true } tokio = { workspace = true} md5 = { workspace = true} +rust-embed = "8.5.0" [dev-dependencies] tempfile = { workspace = true } diff --git a/rust/sqlite/sqlite::memory: b/rust/sqlite/sqlite::memory: deleted file mode 100644 index e249e220ce1f305536affb5d4d624f468c1f355a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 159744 zcmeI5O>i4Wc7QPi36P*9$d+YTqO@vB>!n~3GzR}D#&KX0l2PW51cH{GIE5O_G)S!Q zLt%!LxK(=qDLcs?cdN3MOAa}1RdPvHYAdJI9&*^j9#W~Ty=-bPd&wbNsd6O;m$P|2 zJwHA8BfXY)Dfyu!anRk<^ZI+;uU~h+p*HWYHVjJYoo-7pNYZ!SCrQ3PA;jnNO~BuI z_-lXOfQw1{3;dLX>rs~zzQynCPO-w_&sZ4}{_O0HGyd71%xbggna^i_I`hNn+Vt(< zr@@bcR|85QK6QEW*Z$x5f8`NBEd*LFZI?&3DeM?45=Sd=Z+Nu3eDfr>2^a~AO`KW z7Rc^)yR#q)B4$UUqsds&pY?@*yLxf|=myeZ>l^V^b z*6gSc$vdkX??jDn1S7IM-MctKj6n&kXTM)6SGEePmYN2Ok*&?r`dv~t^fc&cf#^n; z^@Dw-xksbPtAWVf>yoe0*60)cQ4>Z&RoOE-=J#q#c~aHsqiRE|%I@{0{VT!9_3KhE zYl`Y-xL2=vxVY)$R*1j{wAF&H2lbLl@q>z}ss*b6d#DM9tZ!7v`qt`dD7rish&+%- z3G24JMGb{H}@xHF(dfp4JTJTdfiXLAiLRST3$F7dMGhTnGA5 z-n@tOfH&N3o7HMTGiE+dZU8A$TlN^^$@OMOu`^(xuJTl2_z_dx@T_Pkn!Gel&#sID zGCg#7;WbmyM~}RybhrG>a5VZL521Q5Lv!D zsux-}>Q&jky4cIV8jM`MDjgP0jCQ*HGvGmZ!%n{8A{;(OjDuY|v=^zJ;TdYD*`%t` z=(PE5LML^zSSNL8g`#g=3`FkV9@W^E0)3OVm9|Q&&CWJ65}t&q-o;mfk=wVW9*jnq zh5EAlzD{~T^ktef_z3Ga7e=E+bwg?Gvhm@_&h!Z&x{BLiy9b4$=?j5K>E@_rxis`@ zLiqAZ`sI_}CJVP+I83}0jNH5_J-uyG5VC~Jmp!e&A>@qf2_zOHd+o-fJxaElotn~Y z9+(ESt+e_(hfox-5N052icZbq8X#=ztY=u$I3EpmnSn7#guY`JGVL4LScf6DS_HRY zxv;rhSSf-7&}r8j+tqLDopu!_AHlNOqm4r!IufSpvpae`57kR3%Qop$p8{-KUO$m52w17)@w`?PIV5Q@gm1|o&I zQQEOYtIF2JxsT2SBXe_7FKkk_Qmr3nJ%p{qF?>BOGcEGlvw_HFY?LsTL9N>JyJMN1 zlrx7{LcvHZCLKLA*;{IGqbUWGhal|2G<&D+dXdl6H2H32?hv`Wj1XAmC zn!{WPVQ%(LecX8{1_O}~lcV^$mdXKI70jQLIDHrk1S83$^i(n_54mqhQqaSCNC6AC zkYf`UY?PaOI+?rj!H{1dmd`dZ&Sz^iw)xx!u7-Q}Z-hS!{A%j6$qW7m6JcMoe{KEczaNpAU__!H&#mbPx>lQs!zUlJ z%t6fU1+jL=h_zv*zR1^P><8HF%1%QkE#)EA?Io1H=-x!yowj)sbF~&j9xdAzrwZ%+ z-Q$ew@5QrR3WZARo5ga4xf~l#d$@p`;6f}{$Q$HNd1KAVTMDg|H||*rUAsj~p~7kf z_8s^xg`LW)0S~{h1{)Y|^-H06I+aYP6}G3g>wutj#;sbbGOh-a?|K^p>qp; znwQ$M+<2{Fu3waHgX)VO#k;rYq_DC=mN!n4r%ey6!ltv=G;hQe+7*yC5HbRF)ykq#KVMN%@%9t79qVWZi!9&A?6s#Q#{fGTZb4twCZ*J>|`Zh@$+LH}gv=sxkiDN2=W zDwE46)3tgct)(*YloGE~Es@M*WI5BTO@W^89=^r(JjL{!j&*4Z$ixb-A>C)U3Qk(pi5B6Cf`P|MzS6n*=0?`X7yR~!9yFalOg45ax#-ubBRnX zSFb7QdOlN6WNWmpsj`+zXo>I5O|pUb1FJJnvPPz2mY*H7&6O^+)H8FG?wC`0@q^N4 zWz$+G2-`cu_a^Lxhv}Y!r5{tHBfywh>1(m18jne^Z&hRL6#l!zvQ=UFM`kY^=3{-8 zdj+1g-LPE&fCM&c+1%nI8m%i}EbYZB8*ok&EZ<_I4C}6YtA*tvcQc&jCEMnDzp%Om zJG8gvt%|^DV5{Ky=n`YdEMVUgVETl~xW72*fKBZ3nSE=kw4&RNP5;Af;b8bMFQ2!M zKa=fwxbmiUHamz&; zch9^n>)3X1A5Rd*j^pFrlfK?Fpo4>S3#LH6;CA9tSrZADwRy8 z^Kvif4@9H~2Yd$bGl`SpOz$8N?BfVA#F@;w&E8O33Nc%p(BkQAN=t%emh)Nh`HSuryW^j4GPqQ^Q5d?qX1m9s zXul=1WOS$P;io{SV28v+C$y_p>(6WiNqRFb9iDy%FMNK8*wL}6I^w(!X@NYPr@rBpIneg|*-wp4E)$sp> z|1SRok zoFGlT3K3{~bRv9lGBz`ieD2KTB}p>R%4(Bmg)jay!k3BJ*~v>YRTlkTqb(7nVc|ns2_OL^fCP{L5;D%*x!4FKfCP{L560301`j~NB{{S z0VIF~kN^@u0!RP}oJs60301`j~NB{{S0VIF~kN^@u0!RP}oJs%%8&pOKmter2_OL^fCP{L5b(KH4u>f zy$XM?oJwI)C?tRckN^@u0!RP}AOR$R1dsp{KmthMBm_bszwfeD&@cIYXYl<0B-pSF z5Z4r z`;%F1Ha+wC%ui>2I9;2*9sD%-QSfR&3B;!^PyX8f8~=~|iHXlA{zCe#r22mDdzZJr zH+v%(DHNpMbwjB&DQ(rLrZw8z)%squY0xJ|RqLpF<2z$=t}d60g-VfB3h%5I$^7Ul z=5IALQd+MR?-tACUb(bZD1S)aD}J~@^zV>&RyW>>_CnXWG}lH+qjuT`g+GIGS~*va zmBuPKS{gOJKvX%}yF`MKd|v9UjS{AzJsBbhq|O~H1QeRTrPJ+(wm=NpZ7q=9?RIBD z#3E)#qvS!U^6th~g_JiQlva9E*MgCyrEzLg3}uLL8YEvnRytN{G@n|tqdw#cGrkdw z$ntdW;s`{85?IfEzf`Vl6;`cIGgyplZI;&WlDeU%q1QkUF}kc*?kmkb8ckjeMDAXf ze2un7pXiU84TDydJ)>iOueOvYRh>SnHnghjUSHb35{z8GF7>jesBVUP^_qu^n@(

6aO3>FA|C(Dq$}_q^(9&0xOODq%>Li+76U;`(xNlQ_k7pdaPUdq@v>!|k?Ntrj$6 z=JVtRkTSJpk9eM3Z*~+r1I)%MPZf*-G1U#viiV=eOXKwH$|xYyLx&e$GZlUG$csvM z%g+o)qYokx1hZP(*9p8VoJTmF#GB8%rMHLhdetsf460QXgH%du#m!1#?Vi(#$V%}} zVQaNQmbc2Fpi0##!f)PMFWuiN5@s;09=hOa5JUl#Z$<)<<*TE5p>?BPmF=sGz5J`e z$knUTVbR2Br`taR9)vgS~>0sv3<>o8KmMQa6irQioP3 z`qsrj@P$-(d5Qvm+j%t=mL$4-;FR!FuKIv_;aNC8$ z#7n`*&70EG+a?7eOSpX5)A}1i&bXdHVllGUZamtfWV_j^Db41AX+YabtG{yyMF9(8 z2C}B;)GV$6!nV$OhUHZ7(O{Pu7=uLUJ9Z({zLAY}7-FkMaAKAVo6CijA~*n@cD=D( z{kGm|S7GuIESo*rn6wLyM0}XZ=L3=AwNWZ^9$HN}moLfTlieaax12vrp9@B=U6Y>X zP1QIV&PCXx7AJYk;1oE2L6w+8$uJedFUNA{MMhCKYq9w{wJi{aI?6;Ww)MR(CB_ce zNd+7y7{vzJQFPL2HxJ++>ga8&$UFilWiYI95PnkSlx~G5iDnb%?f;&p434gwN9rw%#{%4X7AL;orhvD z5cx1Uimz*_9H3Rf{5gr!hp|8~l1xfZC6n@y`-UV1J*-U|7rG*XJ^iA1#V7tC;xp?_J2QAo%#Fe|DC8#f8_tZw>JES1V$1#jK|Eq z<);_S4LRprQ?J6I&7KF8Z}z^cxA;CBId;0{K`psjgI%R9%vQ#)=34l z%8!mGaiV|b=DCQkA0wg_!7~;i9LX53pH##Xgak!r5^{bHvgJ*b+?=VZj%N=hb>{!6p7I|>lU9UuzvF|rv-ehlZ znWIH-A!|n9kr4yofP6L>a{dlRHn|XtyeCV&$7ZaTQ@wLu$Ij}VK(WDWJ65r@zEb=^ zh>@{-o#%EI7BE-^xdjJZeF16Y!MhMS7>OYDqft@|VNrumv_l$4 z3N73{R%W3b%nqi=LXt<~`8R@*B1lX}N$eco4$5msKaP~wxp%CzPAOYfCqb0->lX_M zPTQY8+gQLrG+=syy-VK=f+?ErEqe!kzmMRm+=&wA)0;I-92eXP;hD@65@S5YVG9wl zorkmyLC0^+!)S&RXsm~@a z_#aG!A?cZaj>t?fBGHfM*7O5itIfpWlaE>9KruH8B-ZX2u{OLnu*hSw*$;@>s_Zm$ z(o!B$-43DF7u}mkyVEvrVxg9cA&;YJM;uil-hbEi8UBEgrAwhuX??R;uCS294W~U^ zz)c7tT&|Ee$er@Wnv=H_S}AYbvjW}i7A=Jes}*<)g1<9ir}Anb!nd#nuT!|yFNNai zR5G1bXgyCe$!t7PtEW>`Nx(Nbok`Z^!{RikXzl2`Tt(ANMY)&|jT(bUSFc*iRPA&% zojl%w_&7U65cCAIEDw%#15tINK+qk;U)DR_7S$F*pw)swSO_cW)QP!phcHaA+f3Ew z+(F>NjLF;6sRs6(bj`<+eMQ~dYYA2Nsa$wI3(hr_&fP@M@)lgg-ywh&I4W_PrTL^R z%Sv3%=JJZB)ibGlMpiR&BCGcDLEw15r*Muz#xW~)aqdccNp8Yc0d#JmPxDfHRv53= zFyoh%ZiDKJ9>u%2=%lc+LY6mHx7OB&#-1*PWHk%Quho=fDx;(likz!ulv=hftNC0W zUfVgM5UD27PhRJ;2bk;`QFeR8+RkoYEAFqtdlW?LKvQ5K6Mb)Yx6_3%OY?moqV7<5 z>4OJV!izy&uI2>-v6l)cSKn(I@P(IEcxl#68twf~bDvs$1olr9(x#8CKBBZWXzT6) z+gCAmcir5wNxgaQsQ;0C6bwpEN6PPDbVxX!?(DerzXQSsf3pJ-rOV`h(EA8^CW9zDrUvm#%yz?3oZ4`9Hl$v;;Z;UX|u9v z#R&_qcMP9F?7(9)JQpnem>L}c#>`4zi;(K^m<0P)HO5ZizdI~j73QwN$Zg)Q@?8P* z$#&-%_ErEu0-LpLzT!hPT35hW+QI8=z&TB@e2a}T#9iN8Ei4zgo8g3)v)5eSFRX6C z8(O#Kt%|^DU{Ub%(Iv)^S-`$0!1M`|aer~r0h`$6GyB$7X+^gkoBoH}!ol!in^wGi z{F&^X2Ui|8MA;`c!Bg1|zr0?Yn7K&@R zXyfjgw`CpM4({U#!g$;~PrB&o2^r>qaD6=?2i%lYE(hk3=ISaalcwT{q?(oGI8CLJ z$#h=s1^t1D^x%Nc0DdNMQk>}>1cH4WA%-}UIk(v_)GURVElz0hbT*|W!7|HvHNjk{ zdM1%p^GQWHx;p`O`t47`T$BkWO2S?nj5N=-!#l=$^5OhN-xhPnKR>o{sq(X*kt@bf zwBM3hGP=|D@V7uu!5b1LDo^0JExWtRycA00<)jK8V>+IU!%`xjW;#rza`9xA=JIml z=r+XQ&lEr24cU|q!mq8Cb!btvSa*)*-Lu1iHyFGiV5$3HD00;F5Y4 z*luwiV`LpOyDp*~*dus4wv#vbsy1^LU}ph*=X$-Cq-oeD({eS~UP$NCa$LzJXfA#D z;9M{^v-~tN&;suKxDuUIW=mM>*F20C_{;pVhd+Fz(^5!FX47e{4m(nb8qLvMGA`Fs zTDGnNYbBM-9F@-oFG(ApTv#*r8f(_}jlJ<0gG1$p3Y($sIf2y`_>r-9_r_6!U_b_T!d9^?X**z%@@PR8ewu zSS#1#aLy2~Wm35sH2Djxy zazaUGFA)^gYA z!@^SCw%O}fZ-D9_?l%ebFT!0(9WC&k5`JuFMt2EoZ>PAq^yU-z=-(RQgmMXSw~kr2 zcd2+KqH#U;H%s+5;#vK&FSp*~Or9A0GGLxyx2Bd->*<7=s?`#iY=$OeEtAW{l~gL8 zujkSkw*H?8|Hc>o4F16n5A(jsR)!O9$18csO-2eap diff --git a/rust/sqlite/src/db.rs b/rust/sqlite/src/db.rs index 522a9c09fcc..c483928013a 100644 --- a/rust/sqlite/src/db.rs +++ b/rust/sqlite/src/db.rs @@ -2,7 +2,6 @@ use crate::config::{MigrationHash, MigrationMode, SqliteDBConfig}; use sha2::{Digest, Sha256}; use sqlx::sqlite::{SqliteConnectOptions, SqlitePool}; use sqlx::{Executor, Row}; -use std::path::PathBuf; // TODO: // - support memory mode, add concurrency tests @@ -15,16 +14,11 @@ struct SqliteDb { impl SqliteDb { pub async fn try_from_config(config: &SqliteDBConfig) -> Result { // TODO: error type - - // PRAGMA legacy_alter_table=ON is required for some of our older - // migrations that use the ALTER TABLE ... RENAME TO ... syntax - // cur.execute("PRAGMA foreign_keys = ON") - // cur.execute("PRAGMA case_sensitive_like = ON") // TODO: copy all other pragmas from python and add basic tests let options = SqliteConnectOptions::new() .filename(&config.url) // Due to a bug in the python code, foreign_keys is turned off - // the python code enabled it in a transaction, however, + // The python code enabled it in a transaction, however, // https://www.sqlite.org/pragma.html states that foreign_keys // is a no-op in a transaction. In order to be able to run our migrations // we turn it off diff --git a/rust/sqlite/src/lib.rs b/rust/sqlite/src/lib.rs index f1377225c73..d6f6a196429 100644 --- a/rust/sqlite/src/lib.rs +++ b/rust/sqlite/src/lib.rs @@ -1,2 +1,3 @@ mod config; mod db; +mod migration_files; diff --git a/rust/sqlite/src/migration_files.rs b/rust/sqlite/src/migration_files.rs new file mode 100644 index 00000000000..89b8974fe98 --- /dev/null +++ b/rust/sqlite/src/migration_files.rs @@ -0,0 +1,16 @@ +use rust_embed::Embed; + +#[derive(Embed)] +#[folder = "../../chromadb/migrations/"] +struct RootMigrationsFolder; + +#[cfg(test)] +mod tests { + use super::*; + + fn test_migration_files() { + for file in RootMigrationsFolder::iter() { + println!("File: {}", file); + } + } +} From 07e7744f3d5a163f295ad2a9b185255b444edd05 Mon Sep 17 00:00:00 2001 From: hammadb Date: Sun, 2 Feb 2025 22:18:11 -0800 Subject: [PATCH 5/7] refactor for cleanliness --- Cargo.lock | 25 ++ rust/sqlite/Cargo.toml | 3 +- rust/sqlite/migrations/__init__.py | 0 .../00001-embeddings.sqlite.sql | 10 + .../00002-embeddings-queue-config.sqlite.sql | 4 + .../00001-embedding-metadata.sqlite.sql | 24 ++ .../00002-embedding-metadata.sqlite.sql | 5 + .../00003-full-text-tokenize.sqlite.sql | 3 + .../metadb/00004-metadata-indices.sqlite.sql | 3 + .../sysdb/00001-collections.sqlite.sql | 15 + .../sysdb/00002-segments.sqlite.sql | 16 + .../00003-collection-dimension.sqlite.sql | 1 + .../sysdb/00004-tenants-databases.sqlite.sql | 29 ++ .../sysdb/00005-remove-topic.sqlite.sql | 4 + ...006-collection-segment-metadata.sqlite.sql | 6 + .../sysdb/00007-collection-config.sqlite.sql | 2 + .../sysdb/00008-maintenance-log.sqlite.sql | 7 + ...009-segment-collection-not-null.sqlite.sql | 11 + rust/sqlite/sqlite::memory: | Bin 0 -> 159744 bytes rust/sqlite/src/config.rs | 4 - rust/sqlite/src/db.rs | 331 +++++------------- rust/sqlite/src/lib.rs | 2 +- rust/sqlite/src/migration_files.rs | 16 - rust/sqlite/src/migrations.rs | 246 +++++++++++++ 24 files changed, 503 insertions(+), 264 deletions(-) create mode 100644 rust/sqlite/migrations/__init__.py create mode 100644 rust/sqlite/migrations/embeddings_queue/00001-embeddings.sqlite.sql create mode 100644 rust/sqlite/migrations/embeddings_queue/00002-embeddings-queue-config.sqlite.sql create mode 100644 rust/sqlite/migrations/metadb/00001-embedding-metadata.sqlite.sql create mode 100644 rust/sqlite/migrations/metadb/00002-embedding-metadata.sqlite.sql create mode 100644 rust/sqlite/migrations/metadb/00003-full-text-tokenize.sqlite.sql create mode 100644 rust/sqlite/migrations/metadb/00004-metadata-indices.sqlite.sql create mode 100644 rust/sqlite/migrations/sysdb/00001-collections.sqlite.sql create mode 100644 rust/sqlite/migrations/sysdb/00002-segments.sqlite.sql create mode 100644 rust/sqlite/migrations/sysdb/00003-collection-dimension.sqlite.sql create mode 100644 rust/sqlite/migrations/sysdb/00004-tenants-databases.sqlite.sql create mode 100644 rust/sqlite/migrations/sysdb/00005-remove-topic.sqlite.sql create mode 100644 rust/sqlite/migrations/sysdb/00006-collection-segment-metadata.sqlite.sql create mode 100644 rust/sqlite/migrations/sysdb/00007-collection-config.sqlite.sql create mode 100644 rust/sqlite/migrations/sysdb/00008-maintenance-log.sqlite.sql create mode 100644 rust/sqlite/migrations/sysdb/00009-segment-collection-not-null.sqlite.sql create mode 100644 rust/sqlite/sqlite::memory: delete mode 100644 rust/sqlite/src/migration_files.rs create mode 100644 rust/sqlite/src/migrations.rs diff --git a/Cargo.lock b/Cargo.lock index 8f0905a3d7c..0a5f86b9518 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1059,6 +1059,16 @@ dependencies = [ "bit-vec 0.4.4", ] +[[package]] +name = "bstr" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "bumpalo" version = "3.15.4" @@ -2686,6 +2696,19 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "globset" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + [[package]] name = "group" version = "0.12.1" @@ -5280,6 +5303,7 @@ version = "8.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e5347777e9aacb56039b0e1f28785929a8a3b709e87482e7442c72e7c12529d" dependencies = [ + "globset", "sha2", "walkdir", ] @@ -5918,6 +5942,7 @@ dependencies = [ "sha2", "sqlx", "tempfile", + "thiserror 1.0.69", "tokio", ] diff --git a/rust/sqlite/Cargo.toml b/rust/sqlite/Cargo.toml index e6ba7944e3f..8aef73436ec 100644 --- a/rust/sqlite/Cargo.toml +++ b/rust/sqlite/Cargo.toml @@ -9,7 +9,8 @@ sha2 = { workspace = true} regex = { workspace = true } tokio = { workspace = true} md5 = { workspace = true} -rust-embed = "8.5.0" +rust-embed = {version = "8.5.0", features = ["include-exclude"]} +thiserror = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/rust/sqlite/migrations/__init__.py b/rust/sqlite/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/rust/sqlite/migrations/embeddings_queue/00001-embeddings.sqlite.sql b/rust/sqlite/migrations/embeddings_queue/00001-embeddings.sqlite.sql new file mode 100644 index 00000000000..078bd897f98 --- /dev/null +++ b/rust/sqlite/migrations/embeddings_queue/00001-embeddings.sqlite.sql @@ -0,0 +1,10 @@ +CREATE TABLE embeddings_queue ( + seq_id INTEGER PRIMARY KEY, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + operation INTEGER NOT NULL, + topic TEXT NOT NULL, + id TEXT NOT NULL, + vector BLOB, + encoding TEXT, + metadata TEXT +); diff --git a/rust/sqlite/migrations/embeddings_queue/00002-embeddings-queue-config.sqlite.sql b/rust/sqlite/migrations/embeddings_queue/00002-embeddings-queue-config.sqlite.sql new file mode 100644 index 00000000000..dde76515414 --- /dev/null +++ b/rust/sqlite/migrations/embeddings_queue/00002-embeddings-queue-config.sqlite.sql @@ -0,0 +1,4 @@ +CREATE TABLE embeddings_queue_config ( + id INTEGER PRIMARY KEY, + config_json_str TEXT +); diff --git a/rust/sqlite/migrations/metadb/00001-embedding-metadata.sqlite.sql b/rust/sqlite/migrations/metadb/00001-embedding-metadata.sqlite.sql new file mode 100644 index 00000000000..cf2e820da64 --- /dev/null +++ b/rust/sqlite/migrations/metadb/00001-embedding-metadata.sqlite.sql @@ -0,0 +1,24 @@ +CREATE TABLE embeddings ( + id INTEGER PRIMARY KEY, + segment_id TEXT NOT NULL, + embedding_id TEXT NOT NULL, + seq_id BLOB NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (segment_id, embedding_id) +); + +CREATE TABLE embedding_metadata ( + id INTEGER REFERENCES embeddings(id), + key TEXT NOT NULL, + string_value TEXT, + int_value INTEGER, + float_value REAL, + PRIMARY KEY (id, key) +); + +CREATE TABLE max_seq_id ( + segment_id TEXT PRIMARY KEY, + seq_id BLOB NOT NULL +); + +CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value); diff --git a/rust/sqlite/migrations/metadb/00002-embedding-metadata.sqlite.sql b/rust/sqlite/migrations/metadb/00002-embedding-metadata.sqlite.sql new file mode 100644 index 00000000000..9684b14ad6d --- /dev/null +++ b/rust/sqlite/migrations/metadb/00002-embedding-metadata.sqlite.sql @@ -0,0 +1,5 @@ +-- SQLite does not support adding check with alter table, as a result, adding a check +-- involve creating a new table and copying the data over. It is over kill with adding +-- a boolean type column. The application write to the table needs to ensure the data +-- integrity. +ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER diff --git a/rust/sqlite/migrations/metadb/00003-full-text-tokenize.sqlite.sql b/rust/sqlite/migrations/metadb/00003-full-text-tokenize.sqlite.sql new file mode 100644 index 00000000000..2b8aa2111ad --- /dev/null +++ b/rust/sqlite/migrations/metadb/00003-full-text-tokenize.sqlite.sql @@ -0,0 +1,3 @@ +CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram'); +INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata; +DROP TABLE embedding_fulltext; diff --git a/rust/sqlite/migrations/metadb/00004-metadata-indices.sqlite.sql b/rust/sqlite/migrations/metadb/00004-metadata-indices.sqlite.sql new file mode 100644 index 00000000000..52bf53a50ea --- /dev/null +++ b/rust/sqlite/migrations/metadb/00004-metadata-indices.sqlite.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS embedding_metadata_int_value ON embedding_metadata (key, int_value) WHERE int_value IS NOT NULL; +CREATE INDEX IF NOT EXISTS embedding_metadata_float_value ON embedding_metadata (key, float_value) WHERE float_value IS NOT NULL; +CREATE INDEX IF NOT EXISTS embedding_metadata_string_value ON embedding_metadata (key, string_value) WHERE string_value IS NOT NULL; diff --git a/rust/sqlite/migrations/sysdb/00001-collections.sqlite.sql b/rust/sqlite/migrations/sysdb/00001-collections.sqlite.sql new file mode 100644 index 00000000000..99abeaab194 --- /dev/null +++ b/rust/sqlite/migrations/sysdb/00001-collections.sqlite.sql @@ -0,0 +1,15 @@ +CREATE TABLE collections ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + topic TEXT NOT NULL, + UNIQUE (name) +); + +CREATE TABLE collection_metadata ( + collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE, + key TEXT NOT NULL, + str_value TEXT, + int_value INTEGER, + float_value REAL, + PRIMARY KEY (collection_id, key) +); diff --git a/rust/sqlite/migrations/sysdb/00002-segments.sqlite.sql b/rust/sqlite/migrations/sysdb/00002-segments.sqlite.sql new file mode 100644 index 00000000000..4f4b8c25d0c --- /dev/null +++ b/rust/sqlite/migrations/sysdb/00002-segments.sqlite.sql @@ -0,0 +1,16 @@ +CREATE TABLE segments ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + scope TEXT NOT NULL, + topic TEXT, + collection TEXT REFERENCES collection(id) +); + +CREATE TABLE segment_metadata ( + segment_id TEXT REFERENCES segments(id) ON DELETE CASCADE, + key TEXT NOT NULL, + str_value TEXT, + int_value INTEGER, + float_value REAL, + PRIMARY KEY (segment_id, key) +); diff --git a/rust/sqlite/migrations/sysdb/00003-collection-dimension.sqlite.sql b/rust/sqlite/migrations/sysdb/00003-collection-dimension.sqlite.sql new file mode 100644 index 00000000000..cb793f49702 --- /dev/null +++ b/rust/sqlite/migrations/sysdb/00003-collection-dimension.sqlite.sql @@ -0,0 +1 @@ +ALTER TABLE collections ADD COLUMN dimension INTEGER; diff --git a/rust/sqlite/migrations/sysdb/00004-tenants-databases.sqlite.sql b/rust/sqlite/migrations/sysdb/00004-tenants-databases.sqlite.sql new file mode 100644 index 00000000000..43372bf97a8 --- /dev/null +++ b/rust/sqlite/migrations/sysdb/00004-tenants-databases.sqlite.sql @@ -0,0 +1,29 @@ +CREATE TABLE IF NOT EXISTS tenants ( + id TEXT PRIMARY KEY, + UNIQUE (id) +); + +CREATE TABLE IF NOT EXISTS databases ( + id TEXT PRIMARY KEY, -- unique globally + name TEXT NOT NULL, -- unique per tenant + tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, + UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name +); + +CREATE TABLE IF NOT EXISTS collections_tmp ( + id TEXT PRIMARY KEY, -- unique globally + name TEXT NOT NULL, -- unique per database + topic TEXT NOT NULL, + dimension INTEGER, + database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE, + UNIQUE (name, database_id) +); + +-- Create default tenant and database +INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs +INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant'); + +INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id) + SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections; +DROP TABLE collections; +ALTER TABLE collections_tmp RENAME TO collections; diff --git a/rust/sqlite/migrations/sysdb/00005-remove-topic.sqlite.sql b/rust/sqlite/migrations/sysdb/00005-remove-topic.sqlite.sql new file mode 100644 index 00000000000..3ed0e028423 --- /dev/null +++ b/rust/sqlite/migrations/sysdb/00005-remove-topic.sqlite.sql @@ -0,0 +1,4 @@ +-- Remove the topic column from the Collections and Segments tables + +ALTER TABLE collections DROP COLUMN topic; +ALTER TABLE segments DROP COLUMN topic; diff --git a/rust/sqlite/migrations/sysdb/00006-collection-segment-metadata.sqlite.sql b/rust/sqlite/migrations/sysdb/00006-collection-segment-metadata.sqlite.sql new file mode 100644 index 00000000000..8d0d5d603d8 --- /dev/null +++ b/rust/sqlite/migrations/sysdb/00006-collection-segment-metadata.sqlite.sql @@ -0,0 +1,6 @@ +-- SQLite does not support adding check with alter table, as a result, adding a check +-- involve creating a new table and copying the data over. It is over kill with adding +-- a boolean type column. The application write to the table needs to ensure the data +-- integrity. +ALTER TABLE collection_metadata ADD COLUMN bool_value INTEGER; +ALTER TABLE segment_metadata ADD COLUMN bool_value INTEGER; diff --git a/rust/sqlite/migrations/sysdb/00007-collection-config.sqlite.sql b/rust/sqlite/migrations/sysdb/00007-collection-config.sqlite.sql new file mode 100644 index 00000000000..35cab052bbb --- /dev/null +++ b/rust/sqlite/migrations/sysdb/00007-collection-config.sqlite.sql @@ -0,0 +1,2 @@ +-- Stores collection configuration dictionaries. +ALTER TABLE collections ADD COLUMN config_json_str TEXT; diff --git a/rust/sqlite/migrations/sysdb/00008-maintenance-log.sqlite.sql b/rust/sqlite/migrations/sysdb/00008-maintenance-log.sqlite.sql new file mode 100644 index 00000000000..8ea44941676 --- /dev/null +++ b/rust/sqlite/migrations/sysdb/00008-maintenance-log.sqlite.sql @@ -0,0 +1,7 @@ +-- Records when database maintenance operations are performed. +-- At time of creation, this table is only used to record vacuum operations. +CREATE TABLE maintenance_log ( + id INT PRIMARY KEY, + timestamp INT NOT NULL, + operation TEXT NOT NULL +); diff --git a/rust/sqlite/migrations/sysdb/00009-segment-collection-not-null.sqlite.sql b/rust/sqlite/migrations/sysdb/00009-segment-collection-not-null.sqlite.sql new file mode 100644 index 00000000000..4f15f8d17a9 --- /dev/null +++ b/rust/sqlite/migrations/sysdb/00009-segment-collection-not-null.sqlite.sql @@ -0,0 +1,11 @@ +-- This makes segments.collection non-nullable. +CREATE TABLE segments_temp ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + scope TEXT NOT NULL, + collection TEXT REFERENCES collection(id) NOT NULL +); + +INSERT INTO segments_temp SELECT * FROM segments; +DROP TABLE segments; +ALTER TABLE segments_temp RENAME TO segments; diff --git a/rust/sqlite/sqlite::memory: b/rust/sqlite/sqlite::memory: new file mode 100644 index 0000000000000000000000000000000000000000..e249e220ce1f305536affb5d4d624f468c1f355a GIT binary patch literal 159744 zcmeI5O>i4Wc7QPi36P*9$d+YTqO@vB>!n~3GzR}D#&KX0l2PW51cH{GIE5O_G)S!Q zLt%!LxK(=qDLcs?cdN3MOAa}1RdPvHYAdJI9&*^j9#W~Ty=-bPd&wbNsd6O;m$P|2 zJwHA8BfXY)Dfyu!anRk<^ZI+;uU~h+p*HWYHVjJYoo-7pNYZ!SCrQ3PA;jnNO~BuI z_-lXOfQw1{3;dLX>rs~zzQynCPO-w_&sZ4}{_O0HGyd71%xbggna^i_I`hNn+Vt(< zr@@bcR|85QK6QEW*Z$x5f8`NBEd*LFZI?&3DeM?45=Sd=Z+Nu3eDfr>2^a~AO`KW z7Rc^)yR#q)B4$UUqsds&pY?@*yLxf|=myeZ>l^V^b z*6gSc$vdkX??jDn1S7IM-MctKj6n&kXTM)6SGEePmYN2Ok*&?r`dv~t^fc&cf#^n; z^@Dw-xksbPtAWVf>yoe0*60)cQ4>Z&RoOE-=J#q#c~aHsqiRE|%I@{0{VT!9_3KhE zYl`Y-xL2=vxVY)$R*1j{wAF&H2lbLl@q>z}ss*b6d#DM9tZ!7v`qt`dD7rish&+%- z3G24JMGb{H}@xHF(dfp4JTJTdfiXLAiLRST3$F7dMGhTnGA5 z-n@tOfH&N3o7HMTGiE+dZU8A$TlN^^$@OMOu`^(xuJTl2_z_dx@T_Pkn!Gel&#sID zGCg#7;WbmyM~}RybhrG>a5VZL521Q5Lv!D zsux-}>Q&jky4cIV8jM`MDjgP0jCQ*HGvGmZ!%n{8A{;(OjDuY|v=^zJ;TdYD*`%t` z=(PE5LML^zSSNL8g`#g=3`FkV9@W^E0)3OVm9|Q&&CWJ65}t&q-o;mfk=wVW9*jnq zh5EAlzD{~T^ktef_z3Ga7e=E+bwg?Gvhm@_&h!Z&x{BLiy9b4$=?j5K>E@_rxis`@ zLiqAZ`sI_}CJVP+I83}0jNH5_J-uyG5VC~Jmp!e&A>@qf2_zOHd+o-fJxaElotn~Y z9+(ESt+e_(hfox-5N052icZbq8X#=ztY=u$I3EpmnSn7#guY`JGVL4LScf6DS_HRY zxv;rhSSf-7&}r8j+tqLDopu!_AHlNOqm4r!IufSpvpae`57kR3%Qop$p8{-KUO$m52w17)@w`?PIV5Q@gm1|o&I zQQEOYtIF2JxsT2SBXe_7FKkk_Qmr3nJ%p{qF?>BOGcEGlvw_HFY?LsTL9N>JyJMN1 zlrx7{LcvHZCLKLA*;{IGqbUWGhal|2G<&D+dXdl6H2H32?hv`Wj1XAmC zn!{WPVQ%(LecX8{1_O}~lcV^$mdXKI70jQLIDHrk1S83$^i(n_54mqhQqaSCNC6AC zkYf`UY?PaOI+?rj!H{1dmd`dZ&Sz^iw)xx!u7-Q}Z-hS!{A%j6$qW7m6JcMoe{KEczaNpAU__!H&#mbPx>lQs!zUlJ z%t6fU1+jL=h_zv*zR1^P><8HF%1%QkE#)EA?Io1H=-x!yowj)sbF~&j9xdAzrwZ%+ z-Q$ew@5QrR3WZARo5ga4xf~l#d$@p`;6f}{$Q$HNd1KAVTMDg|H||*rUAsj~p~7kf z_8s^xg`LW)0S~{h1{)Y|^-H06I+aYP6}G3g>wutj#;sbbGOh-a?|K^p>qp; znwQ$M+<2{Fu3waHgX)VO#k;rYq_DC=mN!n4r%ey6!ltv=G;hQe+7*yC5HbRF)ykq#KVMN%@%9t79qVWZi!9&A?6s#Q#{fGTZb4twCZ*J>|`Zh@$+LH}gv=sxkiDN2=W zDwE46)3tgct)(*YloGE~Es@M*WI5BTO@W^89=^r(JjL{!j&*4Z$ixb-A>C)U3Qk(pi5B6Cf`P|MzS6n*=0?`X7yR~!9yFalOg45ax#-ubBRnX zSFb7QdOlN6WNWmpsj`+zXo>I5O|pUb1FJJnvPPz2mY*H7&6O^+)H8FG?wC`0@q^N4 zWz$+G2-`cu_a^Lxhv}Y!r5{tHBfywh>1(m18jne^Z&hRL6#l!zvQ=UFM`kY^=3{-8 zdj+1g-LPE&fCM&c+1%nI8m%i}EbYZB8*ok&EZ<_I4C}6YtA*tvcQc&jCEMnDzp%Om zJG8gvt%|^DV5{Ky=n`YdEMVUgVETl~xW72*fKBZ3nSE=kw4&RNP5;Af;b8bMFQ2!M zKa=fwxbmiUHamz&; zch9^n>)3X1A5Rd*j^pFrlfK?Fpo4>S3#LH6;CA9tSrZADwRy8 z^Kvif4@9H~2Yd$bGl`SpOz$8N?BfVA#F@;w&E8O33Nc%p(BkQAN=t%emh)Nh`HSuryW^j4GPqQ^Q5d?qX1m9s zXul=1WOS$P;io{SV28v+C$y_p>(6WiNqRFb9iDy%FMNK8*wL}6I^w(!X@NYPr@rBpIneg|*-wp4E)$sp> z|1SRok zoFGlT3K3{~bRv9lGBz`ieD2KTB}p>R%4(Bmg)jay!k3BJ*~v>YRTlkTqb(7nVc|ns2_OL^fCP{L5;D%*x!4FKfCP{L560301`j~NB{{S z0VIF~kN^@u0!RP}oJs60301`j~NB{{S0VIF~kN^@u0!RP}oJs%%8&pOKmter2_OL^fCP{L5b(KH4u>f zy$XM?oJwI)C?tRckN^@u0!RP}AOR$R1dsp{KmthMBm_bszwfeD&@cIYXYl<0B-pSF z5Z4r z`;%F1Ha+wC%ui>2I9;2*9sD%-QSfR&3B;!^PyX8f8~=~|iHXlA{zCe#r22mDdzZJr zH+v%(DHNpMbwjB&DQ(rLrZw8z)%squY0xJ|RqLpF<2z$=t}d60g-VfB3h%5I$^7Ul z=5IALQd+MR?-tACUb(bZD1S)aD}J~@^zV>&RyW>>_CnXWG}lH+qjuT`g+GIGS~*va zmBuPKS{gOJKvX%}yF`MKd|v9UjS{AzJsBbhq|O~H1QeRTrPJ+(wm=NpZ7q=9?RIBD z#3E)#qvS!U^6th~g_JiQlva9E*MgCyrEzLg3}uLL8YEvnRytN{G@n|tqdw#cGrkdw z$ntdW;s`{85?IfEzf`Vl6;`cIGgyplZI;&WlDeU%q1QkUF}kc*?kmkb8ckjeMDAXf ze2un7pXiU84TDydJ)>iOueOvYRh>SnHnghjUSHb35{z8GF7>jesBVUP^_qu^n@(

6aO3>FA|C(Dq$}_q^(9&0xOODq%>Li+76U;`(xNlQ_k7pdaPUdq@v>!|k?Ntrj$6 z=JVtRkTSJpk9eM3Z*~+r1I)%MPZf*-G1U#viiV=eOXKwH$|xYyLx&e$GZlUG$csvM z%g+o)qYokx1hZP(*9p8VoJTmF#GB8%rMHLhdetsf460QXgH%du#m!1#?Vi(#$V%}} zVQaNQmbc2Fpi0##!f)PMFWuiN5@s;09=hOa5JUl#Z$<)<<*TE5p>?BPmF=sGz5J`e z$knUTVbR2Br`taR9)vgS~>0sv3<>o8KmMQa6irQioP3 z`qsrj@P$-(d5Qvm+j%t=mL$4-;FR!FuKIv_;aNC8$ z#7n`*&70EG+a?7eOSpX5)A}1i&bXdHVllGUZamtfWV_j^Db41AX+YabtG{yyMF9(8 z2C}B;)GV$6!nV$OhUHZ7(O{Pu7=uLUJ9Z({zLAY}7-FkMaAKAVo6CijA~*n@cD=D( z{kGm|S7GuIESo*rn6wLyM0}XZ=L3=AwNWZ^9$HN}moLfTlieaax12vrp9@B=U6Y>X zP1QIV&PCXx7AJYk;1oE2L6w+8$uJedFUNA{MMhCKYq9w{wJi{aI?6;Ww)MR(CB_ce zNd+7y7{vzJQFPL2HxJ++>ga8&$UFilWiYI95PnkSlx~G5iDnb%?f;&p434gwN9rw%#{%4X7AL;orhvD z5cx1Uimz*_9H3Rf{5gr!hp|8~l1xfZC6n@y`-UV1J*-U|7rG*XJ^iA1#V7tC;xp?_J2QAo%#Fe|DC8#f8_tZw>JES1V$1#jK|Eq z<);_S4LRprQ?J6I&7KF8Z}z^cxA;CBId;0{K`psjgI%R9%vQ#)=34l z%8!mGaiV|b=DCQkA0wg_!7~;i9LX53pH##Xgak!r5^{bHvgJ*b+?=VZj%N=hb>{!6p7I|>lU9UuzvF|rv-ehlZ znWIH-A!|n9kr4yofP6L>a{dlRHn|XtyeCV&$7ZaTQ@wLu$Ij}VK(WDWJ65r@zEb=^ zh>@{-o#%EI7BE-^xdjJZeF16Y!MhMS7>OYDqft@|VNrumv_l$4 z3N73{R%W3b%nqi=LXt<~`8R@*B1lX}N$eco4$5msKaP~wxp%CzPAOYfCqb0->lX_M zPTQY8+gQLrG+=syy-VK=f+?ErEqe!kzmMRm+=&wA)0;I-92eXP;hD@65@S5YVG9wl zorkmyLC0^+!)S&RXsm~@a z_#aG!A?cZaj>t?fBGHfM*7O5itIfpWlaE>9KruH8B-ZX2u{OLnu*hSw*$;@>s_Zm$ z(o!B$-43DF7u}mkyVEvrVxg9cA&;YJM;uil-hbEi8UBEgrAwhuX??R;uCS294W~U^ zz)c7tT&|Ee$er@Wnv=H_S}AYbvjW}i7A=Jes}*<)g1<9ir}Anb!nd#nuT!|yFNNai zR5G1bXgyCe$!t7PtEW>`Nx(Nbok`Z^!{RikXzl2`Tt(ANMY)&|jT(bUSFc*iRPA&% zojl%w_&7U65cCAIEDw%#15tINK+qk;U)DR_7S$F*pw)swSO_cW)QP!phcHaA+f3Ew z+(F>NjLF;6sRs6(bj`<+eMQ~dYYA2Nsa$wI3(hr_&fP@M@)lgg-ywh&I4W_PrTL^R z%Sv3%=JJZB)ibGlMpiR&BCGcDLEw15r*Muz#xW~)aqdccNp8Yc0d#JmPxDfHRv53= zFyoh%ZiDKJ9>u%2=%lc+LY6mHx7OB&#-1*PWHk%Quho=fDx;(likz!ulv=hftNC0W zUfVgM5UD27PhRJ;2bk;`QFeR8+RkoYEAFqtdlW?LKvQ5K6Mb)Yx6_3%OY?moqV7<5 z>4OJV!izy&uI2>-v6l)cSKn(I@P(IEcxl#68twf~bDvs$1olr9(x#8CKBBZWXzT6) z+gCAmcir5wNxgaQsQ;0C6bwpEN6PPDbVxX!?(DerzXQSsf3pJ-rOV`h(EA8^CW9zDrUvm#%yz?3oZ4`9Hl$v;;Z;UX|u9v z#R&_qcMP9F?7(9)JQpnem>L}c#>`4zi;(K^m<0P)HO5ZizdI~j73QwN$Zg)Q@?8P* z$#&-%_ErEu0-LpLzT!hPT35hW+QI8=z&TB@e2a}T#9iN8Ei4zgo8g3)v)5eSFRX6C z8(O#Kt%|^DU{Ub%(Iv)^S-`$0!1M`|aer~r0h`$6GyB$7X+^gkoBoH}!ol!in^wGi z{F&^X2Ui|8MA;`c!Bg1|zr0?Yn7K&@R zXyfjgw`CpM4({U#!g$;~PrB&o2^r>qaD6=?2i%lYE(hk3=ISaalcwT{q?(oGI8CLJ z$#h=s1^t1D^x%Nc0DdNMQk>}>1cH4WA%-}UIk(v_)GURVElz0hbT*|W!7|HvHNjk{ zdM1%p^GQWHx;p`O`t47`T$BkWO2S?nj5N=-!#l=$^5OhN-xhPnKR>o{sq(X*kt@bf zwBM3hGP=|D@V7uu!5b1LDo^0JExWtRycA00<)jK8V>+IU!%`xjW;#rza`9xA=JIml z=r+XQ&lEr24cU|q!mq8Cb!btvSa*)*-Lu1iHyFGiV5$3HD00;F5Y4 z*luwiV`LpOyDp*~*dus4wv#vbsy1^LU}ph*=X$-Cq-oeD({eS~UP$NCa$LzJXfA#D z;9M{^v-~tN&;suKxDuUIW=mM>*F20C_{;pVhd+Fz(^5!FX47e{4m(nb8qLvMGA`Fs zTDGnNYbBM-9F@-oFG(ApTv#*r8f(_}jlJ<0gG1$p3Y($sIf2y`_>r-9_r_6!U_b_T!d9^?X**z%@@PR8ewu zSS#1#aLy2~Wm35sH2Djxy zazaUGFA)^gYA z!@^SCw%O}fZ-D9_?l%ebFT!0(9WC&k5`JuFMt2EoZ>PAq^yU-z=-(RQgmMXSw~kr2 zcd2+KqH#U;H%s+5;#vK&FSp*~Or9A0GGLxyx2Bd->*<7=s?`#iY=$OeEtAW{l~gL8 zujkSkw*H?8|Hc>o4F16n5A(jsR)!O9$18csO-2eap literal 0 HcmV?d00001 diff --git a/rust/sqlite/src/config.rs b/rust/sqlite/src/config.rs index 446c01bd0c6..8b07e295e0d 100644 --- a/rust/sqlite/src/config.rs +++ b/rust/sqlite/src/config.rs @@ -4,10 +4,6 @@ use std::path::PathBuf; pub(crate) struct SqliteDBConfig { // The SQLite database URL pub(crate) url: String, - // TODO: change this to something bundled with binary - // The root directory where all migration files are stored - // The migration files are stored in subdirectories of this directory - pub(crate) migrations_root_dir: PathBuf, pub(crate) hash_type: MigrationHash, pub(crate) migration_mode: MigrationMode, } diff --git a/rust/sqlite/src/db.rs b/rust/sqlite/src/db.rs index c483928013a..7a9f61a428b 100644 --- a/rust/sqlite/src/db.rs +++ b/rust/sqlite/src/db.rs @@ -1,19 +1,18 @@ -use crate::config::{MigrationHash, MigrationMode, SqliteDBConfig}; -use sha2::{Digest, Sha256}; +use crate::config::{MigrationMode, SqliteDBConfig}; +use crate::migrations::{GetSourceMigrationsError, Migration, MigrationDir, MIGRATION_DIRS}; use sqlx::sqlite::{SqliteConnectOptions, SqlitePool}; use sqlx::{Executor, Row}; +use thiserror::Error; -// TODO: -// - support memory mode, add concurrency tests +// // TODO: +// // - support memory mode, add concurrency tests struct SqliteDb { conn: SqlitePool, config: SqliteDBConfig, - filename_regex: regex::Regex, } impl SqliteDb { - pub async fn try_from_config(config: &SqliteDBConfig) -> Result { - // TODO: error type + pub async fn try_from_config(config: &SqliteDBConfig) -> Result { // TODO: copy all other pragmas from python and add basic tests let options = SqliteConnectOptions::new() .filename(&config.url) @@ -27,30 +26,28 @@ impl SqliteDb { .create_if_missing(true); let conn = SqlitePool::connect_with(options) .await - .map_err(|e| e.to_string())?; - - // TODO: error type - let filename_regex = - regex::Regex::new(r"(\d+)-(.+)\.(.+)\.sql").map_err(|e| e.to_string())?; + .map_err(|e| SqliteCreationError::SqlxError(e))?; let db = Self { conn, config: config.clone(), - filename_regex, }; - db.validate_migrations_root_dir()?; db.initialize_migrations_table().await?; match config.migration_mode { MigrationMode::Apply => { let mut all_unapplied_migrations = Vec::new(); - for dir in migration_dirs.iter() { + for dir in MIGRATION_DIRS.iter() { let applied_migrations = db.get_existing_migrations(dir).await; - let source_migrations = db.get_source_migrations(dir).await?; - let unapplied = db.validate_migrations_and_get_unapplied( - applied_migrations, - source_migrations, - )?; + let source_migrations = dir + .get_source_migrations(&config.hash_type) + .map_err(SqliteCreationError::GetSourceMigrationsError)?; + let unapplied = db + .validate_migrations_and_get_unapplied( + applied_migrations, + source_migrations, + ) + .map_err(SqliteCreationError::MigrationValidationError)?; all_unapplied_migrations.extend(unapplied); } db.apply_migrations(all_unapplied_migrations).await?; @@ -58,17 +55,19 @@ impl SqliteDb { MigrationMode::Validate => { // TODO: Test this if !db.has_initialized_migrations().await { - return Err("Migrations table not initialized".to_string()); + return Err(SqliteCreationError::MigrationsTableNotInitialized); } - for dir in migration_dirs.iter() { + for dir in MIGRATION_DIRS.iter() { let applied_migrations = db.get_existing_migrations(dir).await; - let source_migrations = db.get_source_migrations(dir).await?; + let source_migrations = dir + .get_source_migrations(&config.hash_type) + .map_err(SqliteCreationError::GetSourceMigrationsError)?; let unapplied = db.validate_migrations_and_get_unapplied( applied_migrations, source_migrations, )?; if !unapplied.is_empty() { - return Err("Unapplied migrations found".to_string()); + return Err(SqliteCreationError::UnappliedMigrationsFound); } } } @@ -78,22 +77,18 @@ impl SqliteDb { //////////////////////// Migrations //////////////////////// - // TODO: Real error /// Apply all migrations in a transaction /// Arguments: /// - migrations: Vec - The migrations to apply - async fn apply_migrations(&self, migrations: Vec) -> Result<(), String> { - let mut tx = self.conn.begin().await.map_err(|e| e.to_string())?; + async fn apply_migrations(&self, migrations: Vec) -> Result<(), sqlx::Error> { + let mut tx = self.conn.begin().await?; for migration in migrations { - println!("Applying migration: {}", migration.filename); // Apply the migration - tx.execute("PRAGMA foreign_keys = ON") - .await - .map_err(|e| e.to_string())?; - tx.execute(sqlx::query(&migration.sql)) - .await - .map_err(|e| e.to_string())?; - println!("Applied migration: {}", migration.filename); + // TODO(hammadb): Determine how to handle foreign keys on + // this is copied over from the python code but it does + // not work in a transaction + tx.execute("PRAGMA foreign_keys = ON").await?; + tx.execute(sqlx::query(&migration.sql)).await?; // Bookkeeping let query = r#" @@ -106,25 +101,12 @@ impl SqliteDb { .bind(&migration.filename) .bind(&migration.sql) .bind(&migration.hash); - tx.execute(query).await.map_err(|e| e.to_string())?; + tx.execute(query).await?; } - tx.commit().await.map_err(|e| e.to_string())?; + tx.commit().await?; Ok(()) } - /// Validate the migrations root directory - fn validate_migrations_root_dir(&self) -> Result<(), String> { - // TODO: replace with ChromaError - for dir in migration_dirs.iter() { - let path = self.config.migrations_root_dir.join(dir.as_str()); - if !path.exists() { - return Err(format!("Migration directory {:?} does not exist", path)); - } - } - Ok(()) - } - - // TODO: Real error /// Validate migration sequence and get the migrations that need to be applied /// Arguments: /// - applied_migrations: Vec - The migrations that have been applied, in ascending version order @@ -135,20 +117,20 @@ impl SqliteDb { &self, applied_migrations: Vec, source_migrations: Vec, - ) -> Result, String> { + ) -> Result, MigrationValidationError> { for (db_migration, source_migration) in applied_migrations.iter().zip(source_migrations.iter()) { if db_migration.version != source_migration.version { - return Err(format!( - "Inconsistent version: db={}, source={}", - db_migration.version, source_migration.version + return Err(MigrationValidationError::InconsistentVersion( + db_migration.version, + source_migration.version, )); } if db_migration.hash != source_migration.hash { - return Err(format!( - "Inconsistent hash: db={}, source={}", - db_migration.hash, source_migration.hash + return Err(MigrationValidationError::InconsistentHash( + db_migration.hash.clone(), + source_migration.hash.clone(), )); } } @@ -160,7 +142,7 @@ impl SqliteDb { /// Initialize the migrations table /// Note: /// - This function is idempotent - async fn initialize_migrations_table(&self) -> Result<(), String> { + async fn initialize_migrations_table(&self) -> Result<(), sqlx::Error> { let query = r#" CREATE TABLE IF NOT EXISTS migrations ( dir TEXT NOT NULL, @@ -171,10 +153,7 @@ impl SqliteDb { PRIMARY KEY (dir, version) ) "#; - sqlx::query(query) - .execute(&self.conn) - .await - .map_err(|e| e.to_string())?; + sqlx::query(query).execute(&self.conn).await?; Ok(()) } @@ -222,171 +201,51 @@ impl SqliteDb { let filename: String = row.get("filename"); let sql: String = row.get("sql"); let hash: String = row.get("hash"); - migrations.push(Migration { - dir, - version, - filename, - sql, - hash, - }); + migrations.push(Migration::new(dir, filename, version, sql, hash)); } migrations } - - // TODO: REAL ERROR - /// Get the migrations that are on disk - /// Arguments: - /// - dir: str - The name of the directory that contains the migrations - /// Returns: - /// - Vec - A list of migrations found on disk, sorted by version in ascending order - /// Notes: - /// - Uses the migrations_root_dir of this SqlDB instance - async fn get_source_migrations(&self, dir: &MigrationDir) -> Result, String> { - let on_disk_path = self.config.migrations_root_dir.join(dir.as_str()); - let mut migrations = Vec::new(); - let mut read_dir = tokio::fs::read_dir(on_disk_path) - .await - .map_err(|e| e.to_string())?; - - while let Some(entry) = read_dir.next_entry().await.map_err(|e| e.to_string())? { - let path = entry.path(); - let filename = match path.file_name() { - Some(filename) => filename, - None => return Err("Filename is None".to_string()), - }; - let filename = match filename.to_str() { - Some(filename) => filename, - None => return Err("Filename is not valid".to_string()), - }; - let (version, _) = self.parse_migration_filename(filename)?; - let sql = tokio::fs::read_to_string(&path) - .await - .map_err(|e| e.to_string())?; - let hash = match self.config.hash_type { - MigrationHash::SHA256 => { - let mut hasher = Sha256::new(); - hasher.update(sql.as_bytes()); - format!("{:x}", hasher.finalize()) - } - MigrationHash::MD5 => { - let hash = md5::compute(sql.as_bytes()); - format!("{:x}", hash) - } - }; - migrations.push(Migration { - dir: dir.as_str().to_string(), - version, - filename: filename.to_string(), - sql, - hash, - }); - } - // TODO: Make a Vec wrapper type that enforces sorting - migrations.sort_by(|a, b| a.version.cmp(&b.version)); - Ok(migrations) - } - - // Parse the migration filename - // Arguments: - // - filename: str - The filename of the migration - // Returns: - // - (i32, str) - The version and scope of the migration - // Notes - // - Format is -..sql - // - e.g, 00001-users.sqlite.sql - // - scope is unused, it is legacy from the python implementation. It is - // written but never read - fn parse_migration_filename(&self, filename: &str) -> Result<(i32, String), String> { - let regex_match = self.filename_regex.captures(filename); - let groups = match regex_match { - Some(groups) => groups, - // TODO: Error - None => return Err(format!("Invalid migration filename: {}", filename)), - }; - - // Parse version - let version = match groups.get(1) { - Some(version) => version, - None => return Err("Failed to find version".to_string()), - }; - let version = match version.as_str().parse::() { - Ok(version) => version, - Err(e) => return Err(e.to_string()), - }; - - // Parse scope - let scope = match groups.get(3) { - Some(scope) => scope, - None => return Err("Failed to find scope".to_string()), - }; - let scope = scope.as_str().to_string(); - - Ok((version, scope)) - } } -#[derive(Clone)] -struct Migration { - dir: String, - filename: String, - version: i32, - sql: String, - hash: String, +//////////////////////// Error Types //////////////////////// + +#[derive(Error, Debug)] +pub enum SqliteCreationError { + #[error(transparent)] + SqlxError(#[from] sqlx::Error), + #[error(transparent)] + GetSourceMigrationsError(#[from] GetSourceMigrationsError), + #[error(transparent)] + MigrationValidationError(#[from] MigrationValidationError), + #[error("Migrations table not initialized")] + MigrationsTableNotInitialized, + #[error("Unappliued migrations found")] + UnappliedMigrationsFound, } -enum MigrationDir { - SysDb, - MetaDb, - EmbeddingsQueue, -} - -const migration_dirs: [MigrationDir; 3] = [ - MigrationDir::SysDb, - MigrationDir::MetaDb, - MigrationDir::EmbeddingsQueue, -]; - -impl MigrationDir { - fn as_str(&self) -> &str { - match self { - Self::SysDb => "sysdb", - Self::MetaDb => "metadb", - Self::EmbeddingsQueue => "embeddings_queue", - } - } -} - -//////////////////////// SqliteSysDb //////////////////////// - -struct SqliteSysDb { - conn: SqlitePool, -} - -impl SqliteSysDb { - pub fn new(conn: SqlitePool) -> Self { - Self { conn } - } +#[derive(Error, Debug)] +pub enum MigrationValidationError { + #[error("Inconsistent version: db={0}, source={1}")] + InconsistentVersion(i32, i32), + #[error("Inconsistent hash: db={0}, source={1}")] + InconsistentHash(String, String), } +//////////////////////// Tests //////////////////////// #[cfg(test)] mod tests { use super::*; use crate::config::MigrationHash; use sqlx::Row; - use tempfile::tempdir; + use std::path::PathBuf; //////////////////////// Test Helpers //////////////////////// fn test_migration_dir() -> PathBuf { - let migration_dir = "/Users/hammad/Documents/chroma/chromadb/migrations"; + let migration_dir = "/../migrations".to_string(); PathBuf::from(migration_dir) } - fn existing_test_db_path() -> String { - // TODO: return bundled path - "/Users/hammad/Documents/chroma/chroma/chroma.sqlite3".to_string() - } - fn new_test_db_path() -> String { // TODO: Make tmpfile work // let dir = tempdir().expect("Expect it to be created"); @@ -403,7 +262,6 @@ mod tests { async fn test_sqlite_db() { let config = SqliteDBConfig { url: "sqlite::memory:".to_string(), - migrations_root_dir: test_migration_dir(), hash_type: MigrationHash::MD5, migration_mode: MigrationMode::Apply, }; @@ -423,42 +281,41 @@ mod tests { assert_eq!(name, "migrations"); } - #[tokio::test] - async fn test_migrations_validate_on_existing_db() { - let config: SqliteDBConfig = SqliteDBConfig { - url: existing_test_db_path(), - migrations_root_dir: test_migration_dir(), - hash_type: MigrationHash::MD5, - migration_mode: MigrationMode::Validate, - }; - let db = SqliteDb::try_from_config(&config) - .await - .expect("Expect it to be created"); - - // Check if migrations table exists - let query = r#" - SELECT name FROM sqlite_master WHERE type='table' AND name='migrations' - "#; - let row = sqlx::query(query) - .fetch_one(&db.conn) - .await - .expect("Expect it to be fetched"); - let name: String = row.get("name"); - assert_eq!(name, "migrations"); - } + // #[tokio::test] + // async fn test_migrations_validate_on_existing_db() { + // let config: SqliteDBConfig = SqliteDBConfig { + // url: existing_test_db_path(), + // migrations_root_dir: test_migration_dir(), + // hash_type: MigrationHash::MD5, + // migration_mode: MigrationMode::Validate, + // }; + // let db = SqliteDb::try_from_config(&config) + // .await + // .expect("Expect it to be created"); + + // // Check if migrations table exists + // let query = r#" + // SELECT name FROM sqlite_master WHERE type='table' AND name='migrations' + // "#; + // let row = sqlx::query(query) + // .fetch_one(&db.conn) + // .await + // .expect("Expect it to be fetched"); + // let name: String = row.get("name"); + // assert_eq!(name, "migrations"); + // } #[tokio::test] async fn test_migrations_get_applied_on_new_db() { let config = SqliteDBConfig { url: new_test_db_path(), - migrations_root_dir: test_migration_dir(), hash_type: MigrationHash::MD5, migration_mode: MigrationMode::Apply, }; let db = SqliteDb::try_from_config(&config) .await .expect("Expect it to be created"); - for dir in migration_dirs { + for dir in MIGRATION_DIRS.iter() { let migrations = db.get_existing_migrations(&dir).await; let on_disk_path = test_migration_dir().join(dir.as_str()); // See how many files are in the directory @@ -473,14 +330,4 @@ mod tests { // - tamper with one and test // - add new migration and test // - reorder migrations - - //////////////////////// SqliteSysDb //////////////////////// - - #[tokio::test] - async fn test_sqlite_sysdb() { - let conn = SqlitePool::connect("sqlite::memory:") - .await - .expect("Expect it to be connected"); - let sysdb = SqliteSysDb::new(conn); - } } diff --git a/rust/sqlite/src/lib.rs b/rust/sqlite/src/lib.rs index d6f6a196429..d6d1ebc8b78 100644 --- a/rust/sqlite/src/lib.rs +++ b/rust/sqlite/src/lib.rs @@ -1,3 +1,3 @@ mod config; mod db; -mod migration_files; +mod migrations; diff --git a/rust/sqlite/src/migration_files.rs b/rust/sqlite/src/migration_files.rs deleted file mode 100644 index 89b8974fe98..00000000000 --- a/rust/sqlite/src/migration_files.rs +++ /dev/null @@ -1,16 +0,0 @@ -use rust_embed::Embed; - -#[derive(Embed)] -#[folder = "../../chromadb/migrations/"] -struct RootMigrationsFolder; - -#[cfg(test)] -mod tests { - use super::*; - - fn test_migration_files() { - for file in RootMigrationsFolder::iter() { - println!("File: {}", file); - } - } -} diff --git a/rust/sqlite/src/migrations.rs b/rust/sqlite/src/migrations.rs new file mode 100644 index 00000000000..afd5c003950 --- /dev/null +++ b/rust/sqlite/src/migrations.rs @@ -0,0 +1,246 @@ +use crate::config::MigrationHash; +use core::str; +use regex::Regex; +use rust_embed::Embed; +use sha2::{Digest, Sha256}; +use std::{borrow::Cow, sync::LazyLock}; +use thiserror::Error; + +///////////// Migration Types ////////////// + +// A migration is a single SQL file that is executed to update the database schema +// ## Fields +// - dir: The directory where the migration file is located. One of "sysdb", "metadb", "embeddings_queue" +// - filename: The name of the migration file +// - version: The version of the migration file +// - sql: The SQL content of the migration file +// - hash: The hash of the migration file content +// ## Note +// - Due to legacy naming from the python codebase, the "log" table is known +// as "embeddings_queue" in the Rust codebase. Only in the sql files is it referred to as "embeddings_queue" +// Elsewhere in our code we should refer to it as "log" +#[derive(Clone)] +pub(crate) struct Migration { + pub(crate) dir: String, + pub(crate) filename: String, + pub(crate) version: i32, + pub(crate) sql: String, + pub(crate) hash: String, +} + +impl Migration { + pub(crate) fn new( + dir: String, + filename: String, + version: i32, + sql: String, + hash: String, + ) -> Self { + Self { + dir, + filename, + version, + sql, + hash, + } + } +} + +// A migration dir is a directory that contains migration files +// for a given subsystem +pub(crate) enum MigrationDir { + SysDb, + MetaDb, + EmbeddingsQueue, +} + +pub(crate) const MIGRATION_DIRS: [MigrationDir; 3] = [ + MigrationDir::SysDb, + MigrationDir::MetaDb, + MigrationDir::EmbeddingsQueue, +]; + +#[derive(Error, Debug)] +pub(crate) enum GetSourceMigrationsError { + #[error(transparent)] + ParseMigrationFilenameError(#[from] ParseMigrationFilenameError), + #[error("{0}")] + NoSuchMigrationFile(String), + #[error("Failed to get migration file: {0}")] + FailedToGetMigrationFile(String), +} + +impl MigrationDir { + pub(crate) fn as_str(&self) -> &str { + match self { + Self::SysDb => "sysdb", + Self::MetaDb => "metadb", + Self::EmbeddingsQueue => "embeddings_queue", + } + } + + fn iter(&self) -> Box>> { + match self { + Self::SysDb => Box::new(SysDbMigrationsFolder::iter()), + Self::MetaDb => Box::new(MetaDbMigrationsFolder::iter()), + Self::EmbeddingsQueue => Box::new(EmbeddingsQueueMigrationsFolder::iter()), + } + } + + fn get_file(&self, name: &str) -> Option { + match self { + Self::SysDb => SysDbMigrationsFolder::get(name), + Self::MetaDb => MetaDbMigrationsFolder::get(name), + Self::EmbeddingsQueue => EmbeddingsQueueMigrationsFolder::get(name), + } + } + + /// Get the migrations that are on disk + /// Arguments: + /// - migration_hash: MigrationHash - The hash function to use for the migration files + /// Returns: + /// - Vec - A list of migrations found on disk, sorted by version in ascending order + /// Notes: + /// - Uses the migrations_root_dir of this SqlDB instance + pub(crate) fn get_source_migrations( + &self, + migration_hash: &MigrationHash, + ) -> Result, GetSourceMigrationsError> { + let mut migrations = Vec::new(); + + for migration_name in self.iter() { + let (version, _) = parse_migration_filename(&migration_name) + .map_err(GetSourceMigrationsError::ParseMigrationFilenameError)?; + let sql = match self.get_file(&migration_name) { + Some(sql) => str::from_utf8(&sql.data) + .map_err(|_| { + GetSourceMigrationsError::FailedToGetMigrationFile( + migration_name.to_string(), + ) + })? + .to_string(), + None => { + return Err(GetSourceMigrationsError::NoSuchMigrationFile( + migration_name.to_string(), + )) + } + }; + let hash = match migration_hash { + MigrationHash::SHA256 => { + let mut hasher = Sha256::new(); + hasher.update(sql.as_bytes()); + format!("{:x}", hasher.finalize()) + } + MigrationHash::MD5 => { + let hash = md5::compute(sql.as_bytes()); + format!("{:x}", hash) + } + }; + migrations.push(Migration::new( + self.as_str().to_string(), + migration_name.to_string(), + version, + sql, + hash, + )); + } + + migrations.sort_by(|a, b| a.version.cmp(&b.version)); + Ok(migrations) + } +} + +///////////// MigrationDir Helpers ////////////// + +#[derive(Error, Debug)] +pub(crate) enum ParseMigrationFilenameError { + #[error("Invalid migration filename: {0}")] + InvalidMigrationFilename(String), + #[error("Failed to find version")] + FailedToFindVersion, + #[error("Failed to find scope")] + FailedToFindScope, +} + +// Parse the migration filename +// Arguments: +// - filename: str - The filename of the migration +// Returns: +// - (i32, str) - The version and scope of the migration +// Notes +// - Format is -..sql +// - e.g, 00001-users.sqlite.sql +// - scope is unused, it is legacy from the python implementation. It is +// written but never read +fn parse_migration_filename(filename: &str) -> Result<(i32, String), ParseMigrationFilenameError> { + let regex_match = MIGRATION_FILENAME_REGEX.captures(filename); + let groups = match regex_match { + Some(groups) => groups, + None => { + return Err(ParseMigrationFilenameError::InvalidMigrationFilename( + filename.to_string(), + )) + } + }; + + // Parse version + let version = match groups.get(1) { + Some(version) => version, + None => return Err(ParseMigrationFilenameError::FailedToFindVersion), + }; + let version = match version.as_str().parse::() { + Ok(version) => version, + Err(e) => { + return Err(ParseMigrationFilenameError::InvalidMigrationFilename( + e.to_string(), + )) + } + }; + + // Parse scope + let scope = match groups.get(3) { + Some(scope) => scope, + None => return Err(ParseMigrationFilenameError::FailedToFindScope), + }; + let scope = scope.as_str().to_string(); + + Ok((version, scope)) +} + +static MIGRATION_FILENAME_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r"(\d+)-(.+)\.(.+)\.sql").expect("Failed to compile regex")); + +///////////// Rust Embed Migrations Data ////////////// +// The migration files are embedded in the binary using the `rust_embed` crate +// These are internal to this file and should not be used elsewhere + +#[derive(Embed)] +#[folder = "./migrations/sysdb/"] +#[include = "*.sql"] +struct SysDbMigrationsFolder; + +#[derive(Embed)] +#[folder = "./migrations/metadb/"] +#[include = "*.sql"] +struct MetaDbMigrationsFolder; + +#[derive(Embed)] +#[folder = "./migrations/embeddings_queue/"] +#[include = "*.sql"] +struct EmbeddingsQueueMigrationsFolder; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_migration_files() { + for dir in MIGRATION_DIRS.iter() { + for migration_name in dir.iter() { + let sql = dir + .get_file(&migration_name) + .expect("Failed to get migration file"); + } + } + } +} From e0574af62b8d8e9cebcf96ad78467b43f0cca48e Mon Sep 17 00:00:00 2001 From: hammadb Date: Mon, 3 Feb 2025 02:03:53 -0800 Subject: [PATCH 6/7] tmp --- rust/sqlite/sqlite::memory: | Bin 159744 -> 0 bytes rust/sqlite/src/db.rs | 10 +++------- 2 files changed, 3 insertions(+), 7 deletions(-) delete mode 100644 rust/sqlite/sqlite::memory: diff --git a/rust/sqlite/sqlite::memory: b/rust/sqlite/sqlite::memory: deleted file mode 100644 index e249e220ce1f305536affb5d4d624f468c1f355a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 159744 zcmeI5O>i4Wc7QPi36P*9$d+YTqO@vB>!n~3GzR}D#&KX0l2PW51cH{GIE5O_G)S!Q zLt%!LxK(=qDLcs?cdN3MOAa}1RdPvHYAdJI9&*^j9#W~Ty=-bPd&wbNsd6O;m$P|2 zJwHA8BfXY)Dfyu!anRk<^ZI+;uU~h+p*HWYHVjJYoo-7pNYZ!SCrQ3PA;jnNO~BuI z_-lXOfQw1{3;dLX>rs~zzQynCPO-w_&sZ4}{_O0HGyd71%xbggna^i_I`hNn+Vt(< zr@@bcR|85QK6QEW*Z$x5f8`NBEd*LFZI?&3DeM?45=Sd=Z+Nu3eDfr>2^a~AO`KW z7Rc^)yR#q)B4$UUqsds&pY?@*yLxf|=myeZ>l^V^b z*6gSc$vdkX??jDn1S7IM-MctKj6n&kXTM)6SGEePmYN2Ok*&?r`dv~t^fc&cf#^n; z^@Dw-xksbPtAWVf>yoe0*60)cQ4>Z&RoOE-=J#q#c~aHsqiRE|%I@{0{VT!9_3KhE zYl`Y-xL2=vxVY)$R*1j{wAF&H2lbLl@q>z}ss*b6d#DM9tZ!7v`qt`dD7rish&+%- z3G24JMGb{H}@xHF(dfp4JTJTdfiXLAiLRST3$F7dMGhTnGA5 z-n@tOfH&N3o7HMTGiE+dZU8A$TlN^^$@OMOu`^(xuJTl2_z_dx@T_Pkn!Gel&#sID zGCg#7;WbmyM~}RybhrG>a5VZL521Q5Lv!D zsux-}>Q&jky4cIV8jM`MDjgP0jCQ*HGvGmZ!%n{8A{;(OjDuY|v=^zJ;TdYD*`%t` z=(PE5LML^zSSNL8g`#g=3`FkV9@W^E0)3OVm9|Q&&CWJ65}t&q-o;mfk=wVW9*jnq zh5EAlzD{~T^ktef_z3Ga7e=E+bwg?Gvhm@_&h!Z&x{BLiy9b4$=?j5K>E@_rxis`@ zLiqAZ`sI_}CJVP+I83}0jNH5_J-uyG5VC~Jmp!e&A>@qf2_zOHd+o-fJxaElotn~Y z9+(ESt+e_(hfox-5N052icZbq8X#=ztY=u$I3EpmnSn7#guY`JGVL4LScf6DS_HRY zxv;rhSSf-7&}r8j+tqLDopu!_AHlNOqm4r!IufSpvpae`57kR3%Qop$p8{-KUO$m52w17)@w`?PIV5Q@gm1|o&I zQQEOYtIF2JxsT2SBXe_7FKkk_Qmr3nJ%p{qF?>BOGcEGlvw_HFY?LsTL9N>JyJMN1 zlrx7{LcvHZCLKLA*;{IGqbUWGhal|2G<&D+dXdl6H2H32?hv`Wj1XAmC zn!{WPVQ%(LecX8{1_O}~lcV^$mdXKI70jQLIDHrk1S83$^i(n_54mqhQqaSCNC6AC zkYf`UY?PaOI+?rj!H{1dmd`dZ&Sz^iw)xx!u7-Q}Z-hS!{A%j6$qW7m6JcMoe{KEczaNpAU__!H&#mbPx>lQs!zUlJ z%t6fU1+jL=h_zv*zR1^P><8HF%1%QkE#)EA?Io1H=-x!yowj)sbF~&j9xdAzrwZ%+ z-Q$ew@5QrR3WZARo5ga4xf~l#d$@p`;6f}{$Q$HNd1KAVTMDg|H||*rUAsj~p~7kf z_8s^xg`LW)0S~{h1{)Y|^-H06I+aYP6}G3g>wutj#;sbbGOh-a?|K^p>qp; znwQ$M+<2{Fu3waHgX)VO#k;rYq_DC=mN!n4r%ey6!ltv=G;hQe+7*yC5HbRF)ykq#KVMN%@%9t79qVWZi!9&A?6s#Q#{fGTZb4twCZ*J>|`Zh@$+LH}gv=sxkiDN2=W zDwE46)3tgct)(*YloGE~Es@M*WI5BTO@W^89=^r(JjL{!j&*4Z$ixb-A>C)U3Qk(pi5B6Cf`P|MzS6n*=0?`X7yR~!9yFalOg45ax#-ubBRnX zSFb7QdOlN6WNWmpsj`+zXo>I5O|pUb1FJJnvPPz2mY*H7&6O^+)H8FG?wC`0@q^N4 zWz$+G2-`cu_a^Lxhv}Y!r5{tHBfywh>1(m18jne^Z&hRL6#l!zvQ=UFM`kY^=3{-8 zdj+1g-LPE&fCM&c+1%nI8m%i}EbYZB8*ok&EZ<_I4C}6YtA*tvcQc&jCEMnDzp%Om zJG8gvt%|^DV5{Ky=n`YdEMVUgVETl~xW72*fKBZ3nSE=kw4&RNP5;Af;b8bMFQ2!M zKa=fwxbmiUHamz&; zch9^n>)3X1A5Rd*j^pFrlfK?Fpo4>S3#LH6;CA9tSrZADwRy8 z^Kvif4@9H~2Yd$bGl`SpOz$8N?BfVA#F@;w&E8O33Nc%p(BkQAN=t%emh)Nh`HSuryW^j4GPqQ^Q5d?qX1m9s zXul=1WOS$P;io{SV28v+C$y_p>(6WiNqRFb9iDy%FMNK8*wL}6I^w(!X@NYPr@rBpIneg|*-wp4E)$sp> z|1SRok zoFGlT3K3{~bRv9lGBz`ieD2KTB}p>R%4(Bmg)jay!k3BJ*~v>YRTlkTqb(7nVc|ns2_OL^fCP{L5;D%*x!4FKfCP{L560301`j~NB{{S z0VIF~kN^@u0!RP}oJs60301`j~NB{{S0VIF~kN^@u0!RP}oJs%%8&pOKmter2_OL^fCP{L5b(KH4u>f zy$XM?oJwI)C?tRckN^@u0!RP}AOR$R1dsp{KmthMBm_bszwfeD&@cIYXYl<0B-pSF z5Z4r z`;%F1Ha+wC%ui>2I9;2*9sD%-QSfR&3B;!^PyX8f8~=~|iHXlA{zCe#r22mDdzZJr zH+v%(DHNpMbwjB&DQ(rLrZw8z)%squY0xJ|RqLpF<2z$=t}d60g-VfB3h%5I$^7Ul z=5IALQd+MR?-tACUb(bZD1S)aD}J~@^zV>&RyW>>_CnXWG}lH+qjuT`g+GIGS~*va zmBuPKS{gOJKvX%}yF`MKd|v9UjS{AzJsBbhq|O~H1QeRTrPJ+(wm=NpZ7q=9?RIBD z#3E)#qvS!U^6th~g_JiQlva9E*MgCyrEzLg3}uLL8YEvnRytN{G@n|tqdw#cGrkdw z$ntdW;s`{85?IfEzf`Vl6;`cIGgyplZI;&WlDeU%q1QkUF}kc*?kmkb8ckjeMDAXf ze2un7pXiU84TDydJ)>iOueOvYRh>SnHnghjUSHb35{z8GF7>jesBVUP^_qu^n@(

6aO3>FA|C(Dq$}_q^(9&0xOODq%>Li+76U;`(xNlQ_k7pdaPUdq@v>!|k?Ntrj$6 z=JVtRkTSJpk9eM3Z*~+r1I)%MPZf*-G1U#viiV=eOXKwH$|xYyLx&e$GZlUG$csvM z%g+o)qYokx1hZP(*9p8VoJTmF#GB8%rMHLhdetsf460QXgH%du#m!1#?Vi(#$V%}} zVQaNQmbc2Fpi0##!f)PMFWuiN5@s;09=hOa5JUl#Z$<)<<*TE5p>?BPmF=sGz5J`e z$knUTVbR2Br`taR9)vgS~>0sv3<>o8KmMQa6irQioP3 z`qsrj@P$-(d5Qvm+j%t=mL$4-;FR!FuKIv_;aNC8$ z#7n`*&70EG+a?7eOSpX5)A}1i&bXdHVllGUZamtfWV_j^Db41AX+YabtG{yyMF9(8 z2C}B;)GV$6!nV$OhUHZ7(O{Pu7=uLUJ9Z({zLAY}7-FkMaAKAVo6CijA~*n@cD=D( z{kGm|S7GuIESo*rn6wLyM0}XZ=L3=AwNWZ^9$HN}moLfTlieaax12vrp9@B=U6Y>X zP1QIV&PCXx7AJYk;1oE2L6w+8$uJedFUNA{MMhCKYq9w{wJi{aI?6;Ww)MR(CB_ce zNd+7y7{vzJQFPL2HxJ++>ga8&$UFilWiYI95PnkSlx~G5iDnb%?f;&p434gwN9rw%#{%4X7AL;orhvD z5cx1Uimz*_9H3Rf{5gr!hp|8~l1xfZC6n@y`-UV1J*-U|7rG*XJ^iA1#V7tC;xp?_J2QAo%#Fe|DC8#f8_tZw>JES1V$1#jK|Eq z<);_S4LRprQ?J6I&7KF8Z}z^cxA;CBId;0{K`psjgI%R9%vQ#)=34l z%8!mGaiV|b=DCQkA0wg_!7~;i9LX53pH##Xgak!r5^{bHvgJ*b+?=VZj%N=hb>{!6p7I|>lU9UuzvF|rv-ehlZ znWIH-A!|n9kr4yofP6L>a{dlRHn|XtyeCV&$7ZaTQ@wLu$Ij}VK(WDWJ65r@zEb=^ zh>@{-o#%EI7BE-^xdjJZeF16Y!MhMS7>OYDqft@|VNrumv_l$4 z3N73{R%W3b%nqi=LXt<~`8R@*B1lX}N$eco4$5msKaP~wxp%CzPAOYfCqb0->lX_M zPTQY8+gQLrG+=syy-VK=f+?ErEqe!kzmMRm+=&wA)0;I-92eXP;hD@65@S5YVG9wl zorkmyLC0^+!)S&RXsm~@a z_#aG!A?cZaj>t?fBGHfM*7O5itIfpWlaE>9KruH8B-ZX2u{OLnu*hSw*$;@>s_Zm$ z(o!B$-43DF7u}mkyVEvrVxg9cA&;YJM;uil-hbEi8UBEgrAwhuX??R;uCS294W~U^ zz)c7tT&|Ee$er@Wnv=H_S}AYbvjW}i7A=Jes}*<)g1<9ir}Anb!nd#nuT!|yFNNai zR5G1bXgyCe$!t7PtEW>`Nx(Nbok`Z^!{RikXzl2`Tt(ANMY)&|jT(bUSFc*iRPA&% zojl%w_&7U65cCAIEDw%#15tINK+qk;U)DR_7S$F*pw)swSO_cW)QP!phcHaA+f3Ew z+(F>NjLF;6sRs6(bj`<+eMQ~dYYA2Nsa$wI3(hr_&fP@M@)lgg-ywh&I4W_PrTL^R z%Sv3%=JJZB)ibGlMpiR&BCGcDLEw15r*Muz#xW~)aqdccNp8Yc0d#JmPxDfHRv53= zFyoh%ZiDKJ9>u%2=%lc+LY6mHx7OB&#-1*PWHk%Quho=fDx;(likz!ulv=hftNC0W zUfVgM5UD27PhRJ;2bk;`QFeR8+RkoYEAFqtdlW?LKvQ5K6Mb)Yx6_3%OY?moqV7<5 z>4OJV!izy&uI2>-v6l)cSKn(I@P(IEcxl#68twf~bDvs$1olr9(x#8CKBBZWXzT6) z+gCAmcir5wNxgaQsQ;0C6bwpEN6PPDbVxX!?(DerzXQSsf3pJ-rOV`h(EA8^CW9zDrUvm#%yz?3oZ4`9Hl$v;;Z;UX|u9v z#R&_qcMP9F?7(9)JQpnem>L}c#>`4zi;(K^m<0P)HO5ZizdI~j73QwN$Zg)Q@?8P* z$#&-%_ErEu0-LpLzT!hPT35hW+QI8=z&TB@e2a}T#9iN8Ei4zgo8g3)v)5eSFRX6C z8(O#Kt%|^DU{Ub%(Iv)^S-`$0!1M`|aer~r0h`$6GyB$7X+^gkoBoH}!ol!in^wGi z{F&^X2Ui|8MA;`c!Bg1|zr0?Yn7K&@R zXyfjgw`CpM4({U#!g$;~PrB&o2^r>qaD6=?2i%lYE(hk3=ISaalcwT{q?(oGI8CLJ z$#h=s1^t1D^x%Nc0DdNMQk>}>1cH4WA%-}UIk(v_)GURVElz0hbT*|W!7|HvHNjk{ zdM1%p^GQWHx;p`O`t47`T$BkWO2S?nj5N=-!#l=$^5OhN-xhPnKR>o{sq(X*kt@bf zwBM3hGP=|D@V7uu!5b1LDo^0JExWtRycA00<)jK8V>+IU!%`xjW;#rza`9xA=JIml z=r+XQ&lEr24cU|q!mq8Cb!btvSa*)*-Lu1iHyFGiV5$3HD00;F5Y4 z*luwiV`LpOyDp*~*dus4wv#vbsy1^LU}ph*=X$-Cq-oeD({eS~UP$NCa$LzJXfA#D z;9M{^v-~tN&;suKxDuUIW=mM>*F20C_{;pVhd+Fz(^5!FX47e{4m(nb8qLvMGA`Fs zTDGnNYbBM-9F@-oFG(ApTv#*r8f(_}jlJ<0gG1$p3Y($sIf2y`_>r-9_r_6!U_b_T!d9^?X**z%@@PR8ewu zSS#1#aLy2~Wm35sH2Djxy zazaUGFA)^gYA z!@^SCw%O}fZ-D9_?l%ebFT!0(9WC&k5`JuFMt2EoZ>PAq^yU-z=-(RQgmMXSw~kr2 zcd2+KqH#U;H%s+5;#vK&FSp*~Or9A0GGLxyx2Bd->*<7=s?`#iY=$OeEtAW{l~gL8 zujkSkw*H?8|Hc>o4F16n5A(jsR)!O9$18csO-2eap diff --git a/rust/sqlite/src/db.rs b/rust/sqlite/src/db.rs index 0760ffe2fda..19a335b5675 100644 --- a/rust/sqlite/src/db.rs +++ b/rust/sqlite/src/db.rs @@ -241,6 +241,7 @@ mod tests { use crate::config::MigrationHash; use sqlx::Row; use std::path::PathBuf; + use tempfile::tempdir; //////////////////////// Test Helpers //////////////////////// @@ -250,13 +251,8 @@ mod tests { } fn new_test_db_path() -> String { - // TODO: Make tmpfile work - // let dir = tempdir().expect("Expect it to be created"); - // let path = dir.path().join("chroma.sqlite3"); - let path = "/Users/hammad/Documents/chroma/chroma/chromaTEST.sqlite3".to_string(); - // remove the file if it exists - std::fs::remove_file(&path).unwrap_or_default(); - path + let path = tempdir().unwrap().into_path().join("test.db"); + path.to_str().unwrap().to_string() } //////////////////////// SqliteDb //////////////////////// From b8a841cefe715eeed10876560a56b778f233cce4 Mon Sep 17 00:00:00 2001 From: hammadb Date: Mon, 3 Feb 2025 02:04:40 -0800 Subject: [PATCH 7/7] cln --- rust/sqlite/src/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/sqlite/src/db.rs b/rust/sqlite/src/db.rs index 19a335b5675..d06c12398c5 100644 --- a/rust/sqlite/src/db.rs +++ b/rust/sqlite/src/db.rs @@ -260,7 +260,7 @@ mod tests { #[tokio::test] async fn test_sqlite_db() { let config = SqliteDBConfig { - url: "sqlite::memory:".to_string(), + url: new_test_db_path(), hash_type: MigrationHash::MD5, migration_mode: MigrationMode::Apply, };