From 88e4d9f229f3613278d2b94a63d93d28cecec579 Mon Sep 17 00:00:00 2001 From: frectonz Date: Sat, 22 Jun 2024 01:54:18 +0300 Subject: [PATCH] feat: initial duckdb support --- .gitignore | 1 + Cargo.lock | 463 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/main.rs | 384 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 835 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 2c4918c..ea8d2b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target *.db +*.duckdb diff --git a/Cargo.lock b/Cargo.lock index b4bef38..8296d69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,6 +46,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "const-random", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -142,6 +144,169 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "arrow" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num", +] + +[[package]] +name = "arrow-array" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" +dependencies = [ + "ahash 0.8.11", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.14.5", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ord" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num", +] + +[[package]] +name = "arrow-row" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" +dependencies = [ + "ahash 0.8.11", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", + "hashbrown 0.14.5", +] + +[[package]] +name = "arrow-schema" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" +dependencies = [ + "bitflags 2.5.0", +] + +[[package]] +name = "arrow-select" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" +dependencies = [ + "ahash 0.8.11", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax 0.8.3", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -175,6 +340,15 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "autocfg" version = "1.3.0" @@ -416,6 +590,12 @@ dependencies = [ "serde", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cbc" version = "0.1.2" @@ -575,6 +755,37 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "comfy-table" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" +dependencies = [ + "strum 0.26.2", + "strum_macros 0.26.4", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -665,6 +876,12 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -736,6 +953,24 @@ dependencies = [ "subtle", ] +[[package]] +name = "duckdb" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "424ede399a5d1084e65c0888fda71e407e5809400c92ff2cf510bfd1697b9c76" +dependencies = [ + "arrow", + "cast", + "fallible-iterator 0.3.0", + "fallible-streaming-iterator", + "hashlink 0.8.4", + "libduckdb-sys", + "memchr", + "rust_decimal", + "smallvec", + "strum 0.25.0", +] + [[package]] name = "either" version = "1.12.0" @@ -795,6 +1030,18 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", +] + [[package]] name = "flate2" version = "1.0.30" @@ -1019,6 +1266,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1375,12 +1633,92 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "libduckdb-sys" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b3f02cecc430f61561bde538d42af4be2d9d5a8b058f74883e460bc1055461" +dependencies = [ + "autocfg", + "cc", + "flate2", + "pkg-config", + "serde", + "serde_json", + "tar", + "vcpkg", +] + [[package]] name = "libloading" version = "0.8.3" @@ -1771,6 +2109,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.5" @@ -1781,6 +2133,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -1796,6 +2157,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1803,6 +2186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2599,6 +2983,7 @@ dependencies = [ "chrono", "clap", "color-eyre", + "duckdb", "futures", "include_dir", "libsql", @@ -2638,6 +3023,47 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +dependencies = [ + "strum_macros 0.25.3", +] + +[[package]] +name = "strum" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" + +[[package]] +name = "strum_macros" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.66", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.66", +] + [[package]] name = "subprocess" version = "0.2.9" @@ -2700,6 +3126,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -2768,6 +3205,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -3187,6 +3633,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" +[[package]] +name = "unicode-width" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + [[package]] name = "untrusted" version = "0.9.0" @@ -3654,6 +4106,17 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "zerocopy" version = "0.7.34" diff --git a/Cargo.toml b/Cargo.toml index 2ec2f4b..84c35ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ futures = "0.3.30" libsql = { git = "https://github.com/tursodatabase/libsql.git", features = ["remote"] } tokio-postgres = "0.7.10" mysql_async = { version = "0.34.1", default-features = false, features = ["rustls-tls", "default-rustls"] } +duckdb = { version = "0.10.2", features = ["bundled"] } [profile.release] strip = true diff --git a/src/main.rs b/src/main.rs index 868c83d..d2e7ec7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,6 +45,12 @@ enum Command { /// mysql connection url [mysql://user:password@localhost/sample] url: String, }, + + /// A local DuckDB database. + Duckdb { + /// Path to the the duckdb file. + database: String, + }, } #[tokio::main] @@ -72,6 +78,7 @@ async fn main() -> color_eyre::Result<()> { } Command::Postgres { url } => AllDbs::Postgres(postgres::Db::open(url).await?), Command::Mysql { url } => AllDbs::Mysql(mysql::Db::open(url).await?), + Command::Duckdb { database } => AllDbs::Duckdb(duckdb::Db::open(database).await?), }; let cors = warp::cors() @@ -177,6 +184,7 @@ enum AllDbs { Libsql(libsql::Db), Postgres(postgres::Db), Mysql(mysql::Db), + Duckdb(duckdb::Db), } #[async_trait] @@ -187,6 +195,7 @@ impl Database for AllDbs { AllDbs::Libsql(x) => x.overview().await, AllDbs::Postgres(x) => x.overview().await, AllDbs::Mysql(x) => x.overview().await, + AllDbs::Duckdb(x) => x.overview().await, } } @@ -196,6 +205,7 @@ impl Database for AllDbs { AllDbs::Libsql(x) => x.tables().await, AllDbs::Postgres(x) => x.tables().await, AllDbs::Mysql(x) => x.tables().await, + AllDbs::Duckdb(x) => x.tables().await, } } @@ -205,6 +215,7 @@ impl Database for AllDbs { AllDbs::Libsql(x) => x.table(name).await, AllDbs::Postgres(x) => x.table(name).await, AllDbs::Mysql(x) => x.table(name).await, + AllDbs::Duckdb(x) => x.table(name).await, } } @@ -218,6 +229,7 @@ impl Database for AllDbs { AllDbs::Libsql(x) => x.table_data(name, page).await, AllDbs::Postgres(x) => x.table_data(name, page).await, AllDbs::Mysql(x) => x.table_data(name, page).await, + AllDbs::Duckdb(x) => x.table_data(name, page).await, } } @@ -227,6 +239,7 @@ impl Database for AllDbs { AllDbs::Libsql(x) => x.query(query).await, AllDbs::Postgres(x) => x.query(query).await, AllDbs::Mysql(x) => x.query(query).await, + AllDbs::Duckdb(x) => x.query(query).await, } } } @@ -1607,9 +1620,328 @@ mod mysql { } } +mod duckdb { + use async_trait::async_trait; + use color_eyre::eyre; + use color_eyre::eyre::OptionExt; + use duckdb::{Config, Connection}; + use std::{ + path::Path, + sync::{Arc, Mutex}, + }; + + use crate::{ + helpers, + responses::{self, RowCount}, + Database, ROWS_PER_PAGE, + }; + + #[derive(Clone)] + pub struct Db { + path: String, + conn: Arc>, + } + + impl Db { + pub async fn open(path: String) -> color_eyre::Result { + let p = path.to_owned(); + let conn = tokio::task::spawn_blocking(move || { + let config = Config::default().access_mode(duckdb::AccessMode::ReadOnly)?; + let conn = Connection::open_with_flags(p, config)?; + + eyre::Ok(conn) + }) + .await??; + + let c = conn.try_clone()?; + let tables = tokio::task::spawn_blocking(move || { + let tables: i32 = c.query_row( + r#" + SELECT count(*) + FROM information_schema.tables + WHERE table_schema = 'main' AND table_type = 'BASE TABLE' + "#, + [], + |row| row.get(0), + )?; + + eyre::Ok(tables) + }) + .await??; + + tracing::info!( + "found {tables} table{} in {path}", + if tables == 1 { "" } else { "s" } + ); + Ok(Self { + path, + conn: Arc::new(Mutex::new(conn)), + }) + } + } + + #[async_trait] + impl Database for Db { + async fn overview(&self) -> color_eyre::Result { + let file_name = Path::new(&self.path) + .file_name() + .ok_or_eyre("failed to get file name overview")? + .to_str() + .ok_or_eyre("file name is not utf-8")? + .to_owned(); + + let metadata = tokio::fs::metadata(&self.path).await?; + + let file_size = helpers::format_size(metadata.len() as f64); + let modified = Some(metadata.modified()?.into()); + let created = metadata.created().ok().map(Into::into); + + let c = self.conn.clone(); + let (tables, indexes, triggers, views, counts) = + tokio::task::spawn_blocking(move || { + let c = c.lock().expect("could not get lock on connection"); + + let tables: i32 = c.query_row( + r#" + SELECT count(*) + FROM information_schema.tables + WHERE table_schema = 'main' AND table_type = 'BASE TABLE' + "#, + [], + |row| row.get(0), + )?; + + let indexes: i32 = + c.query_row("SELECT count(*) FROM duckdb_indexes;", [], |row| row.get(0))?; + + let triggers: i32 = c.query_row( + r#" + SELECT count(*) + FROM duckdb_constraints + WHERE constraint_type = 'TRIGGER' + "#, + [], + |row| row.get(0), + )?; + + let views: i32 = c.query_row( + r#" + SELECT count(*) + FROM information_schema.tables + WHERE table_type = 'VIEW' + "#, + [], + |row| row.get(0), + )?; + + let mut table_names_stmt = c.prepare( + r#" + SELECT table_name + FROM information_schema.tables + WHERE table_type = 'BASE TABLE' + "#, + )?; + let table_names = table_names_stmt + .query_map([], |row| row.get(0))? + .filter_map(|n| n.ok()) + .collect::>(); + + let mut counts = Vec::with_capacity(table_names.len()); + for name in table_names { + let count: i32 = + c.query_row(&format!(r#"SELECT count(*) FROM "{name}""#), [], |row| { + row.get(0) + })?; + + counts.push(RowCount { name, count }); + } + + eyre::Ok((tables, indexes, triggers, views, counts)) + }) + .await??; + + Ok(responses::Overview { + file_name, + sqlite_version: None, + file_size, + created, + modified, + tables, + indexes, + triggers, + views, + counts, + }) + } + + async fn tables(&self) -> color_eyre::Result { + let c = self.conn.clone(); + let tables = tokio::task::spawn_blocking(move || { + let c = c.lock().expect("could not get lock on connection"); + + let mut table_names_stmt = c.prepare( + r#" + SELECT table_name + FROM information_schema.tables + WHERE table_type = 'BASE TABLE' + "#, + )?; + let table_names = table_names_stmt + .query_map([], |row| row.get(0))? + .filter_map(|n| n.ok()) + .collect::>(); + + let mut counts = Vec::with_capacity(table_names.len()); + for name in table_names { + let count: i32 = + c.query_row(&format!(r#"SELECT count(*) FROM "{name}""#), [], |row| { + row.get(0) + })?; + + counts.push(RowCount { name, count }); + } + + eyre::Ok(counts) + }) + .await??; + + Ok(responses::Tables { tables }) + } + + async fn table(&self, name: String) -> color_eyre::Result { + let c = self.conn.clone(); + + let (name, sql, row_count, table_size, index_count, column_count) = + tokio::task::spawn_blocking(move || { + let c = c.lock().expect("could not get lock on connection"); + + let sql = None; + + let row_count: i32 = + c.query_row(&format!(r#"SELECT count(*) FROM "{name}""#), [], |row| { + row.get(0) + })?; + + let table_size: i64 = c.query_row( + "SELECT estimated_size FROM duckdb_tables WHERE table_name = ?", + [&name], + |row| row.get(0), + )?; + let table_size = helpers::format_size(table_size as f64); + + let index_count: i32 = c.query_row( + "SELECT index_count FROM duckdb_tables WHERE table_name = ?", + [&name], + |row| row.get(0), + )?; + + let column_count: i32 = c.query_row( + "SELECT column_count FROM duckdb_tables WHERE table_name = ?", + [&name], + |row| row.get(0), + )?; + + eyre::Ok((name, sql, row_count, table_size, index_count, column_count)) + }) + .await??; + + Ok(responses::Table { + name, + sql, + row_count, + table_size, + index_count, + column_count, + }) + } + + async fn table_data( + &self, + name: String, + page: i32, + ) -> color_eyre::Result { + let c = self.conn.clone(); + + let (columns, rows) = tokio::task::spawn_blocking(move || { + let c = c.lock().expect("could not get lock on connection"); + + let first_column: String = + c.query_row(&format!("PRAGMA table_info('{name}')"), [], |row| { + row.get(1) + })?; + + let offset = (page - 1) * ROWS_PER_PAGE; + let sql = format!( + r#" + SELECT * FROM "{name}" + ORDER BY "{first_column}" + LIMIT {ROWS_PER_PAGE} + OFFSET {offset}; + "# + ); + let mut stmt = c.prepare(&sql)?; + + let rows = stmt + .query_map([], |r| { + let mut rows = Vec::new(); + let mut index = 0; + + while let Ok(val) = r.get_ref(index) { + let val = helpers::duckdb_value_to_json(val); + rows.push(val); + index += 1; + } + + Ok(rows) + })? + .filter_map(|r| r.ok()) + .collect::>(); + + let columns = stmt.column_names(); + + eyre::Ok((columns, rows)) + }) + .await??; + + Ok(responses::TableData { columns, rows }) + } + + async fn query(&self, query: String) -> color_eyre::Result { + let c = self.conn.clone(); + + let (columns, rows) = tokio::task::spawn_blocking(move || { + let c = c.lock().expect("could not get lock on connection"); + + let mut stmt = c.prepare(&query)?; + let columns = stmt.column_names(); + + let columns_len = columns.len(); + let rows = stmt + .query_map([], |r| { + let mut rows = Vec::with_capacity(columns_len); + for i in 0..columns_len { + let val = serde_json::Value::String(r.get_ref(i)?.as_str()?.to_owned()); + rows.push(val); + } + + Ok(rows) + })? + .filter_map(|r| r.ok()) + .collect::>(); + + eyre::Ok((columns, rows)) + }) + .await??; + + Ok(responses::Query { columns, rows }) + } + } +} + mod helpers { - use libsql::Value; - use tokio_rusqlite::types::ValueRef; + use duckdb::types::ValueRef as DuckdbValue; + use libsql::Value as LibsqlValue; + use tokio_rusqlite::types::ValueRef as SqliteValue; pub fn format_size(mut size: f64) -> String { const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; @@ -1623,23 +1955,47 @@ mod helpers { format!("{:.2} {}", size, UNITS[unit]) } - pub fn rusqlite_value_to_json(v: ValueRef) -> serde_json::Value { + pub fn rusqlite_value_to_json(v: SqliteValue) -> serde_json::Value { + use SqliteValue::*; + match v { + Null => serde_json::Value::Null, + Integer(x) => serde_json::json!(x), + Real(x) => serde_json::json!(x), + Text(s) => serde_json::Value::String(String::from_utf8_lossy(s).into_owned()), + Blob(s) => serde_json::json!(s), + } + } + + pub fn libsql_value_to_json(v: LibsqlValue) -> serde_json::Value { + use LibsqlValue::*; match v { - ValueRef::Null => serde_json::Value::Null, - ValueRef::Integer(x) => serde_json::json!(x), - ValueRef::Real(x) => serde_json::json!(x), - ValueRef::Text(s) => serde_json::Value::String(String::from_utf8_lossy(s).into_owned()), - ValueRef::Blob(s) => serde_json::json!(s), + Null => serde_json::Value::Null, + Integer(x) => serde_json::json!(x), + Real(x) => serde_json::json!(x), + Text(s) => serde_json::Value::String(s), + Blob(s) => serde_json::json!(s), } } - pub fn libsql_value_to_json(v: Value) -> serde_json::Value { + pub fn duckdb_value_to_json(v: DuckdbValue) -> serde_json::Value { + use DuckdbValue::*; match v { - Value::Null => serde_json::Value::Null, - Value::Integer(x) => serde_json::json!(x), - Value::Real(x) => serde_json::json!(x), - Value::Text(s) => serde_json::Value::String(s), - Value::Blob(s) => serde_json::json!(s), + Null => serde_json::Value::Null, + Boolean(b) => serde_json::Value::Bool(b), + TinyInt(x) => serde_json::json!(x), + SmallInt(x) => serde_json::json!(x), + Int(x) => serde_json::json!(x), + BigInt(x) => serde_json::json!(x), + HugeInt(x) => serde_json::json!(x), + UTinyInt(x) => serde_json::json!(x), + USmallInt(x) => serde_json::json!(x), + UInt(x) => serde_json::json!(x), + UBigInt(x) => serde_json::json!(x), + Float(x) => serde_json::json!(x), + Double(x) => serde_json::json!(x), + Decimal(x) => serde_json::json!(x), + Text(_) => serde_json::Value::String(v.as_str().unwrap().to_owned()), + v => serde_json::Value::String(format!("{v:?}")), } } }