diff --git a/Cargo.lock b/Cargo.lock index 9fb3c20..a5205ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -558,6 +558,15 @@ dependencies = [ "syn_derive", ] +[[package]] +name = "bstr" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +dependencies = [ + "memchr", +] + [[package]] name = "btoi" version = "0.4.3" @@ -733,6 +742,49 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +[[package]] +name = "clickhouse" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0875e527e299fc5f4faba42870bf199a39ab0bb2dbba1b8aef0a2151451130f" +dependencies = [ + "bstr", + "bytes", + "clickhouse-derive", + "clickhouse-rs-cityhash-sys", + "futures", + "hyper", + "hyper-tls", + "lz4", + "sealed", + "serde", + "static_assertions", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "clickhouse-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18af5425854858c507eec70f7deb4d5d8cec4216fcb086283a78872387281ea5" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "clickhouse-rs-cityhash-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4baf9d4700a28d6cb600e17ed6ae2b43298a5245f1f76b4eab63027ebfd592b9" +dependencies = [ + "cc", +] + [[package]] name = "cmake" version = "0.1.50" @@ -1050,6 +1102,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + [[package]] name = "filetime" version = "0.2.23" @@ -1078,6 +1136,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1358,6 +1431,15 @@ dependencies = [ "http 0.2.12", ] +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.4.1" @@ -1505,6 +1587,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -1928,6 +2023,26 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "lz4" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6eab492fe7f8651add23237ea56dbf11b3c4ff762ab83d40a47f11433421f91" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9764018d143cc854c9f17f0b907de70f14393b1f502da6375dce70f00514eb3" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchers" version = "0.1.0" @@ -2114,6 +2229,23 @@ dependencies = [ "zstd", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.3" @@ -2249,12 +2381,50 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "overload" version = "0.1.1" @@ -2845,6 +3015,18 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "sealed" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b5e421024b5e5edfbaa8e60ecf90bda9dbffc602dbb230e6028763f85f0c68c" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "security-framework" version = "2.11.0" @@ -2888,6 +3070,17 @@ dependencies = [ "syn 2.0.67", ] +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "serde_json" version = "1.0.117" @@ -3007,6 +3200,7 @@ dependencies = [ "async-trait", "chrono", "clap", + "clickhouse", "color-eyre", "duckdb", "futures", @@ -3163,6 +3357,18 @@ dependencies = [ "xattr", ] +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -3295,6 +3501,16 @@ dependencies = [ "syn 2.0.67", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.10" @@ -3659,6 +3875,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + [[package]] name = "unicode-width" version = "0.1.13" diff --git a/Cargo.toml b/Cargo.toml index 0830f16..2bc6990 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ tokio-postgres = "0.7.10" mysql_async = { version = "0.34.1", default-features = false, features = ["rustls-tls", "default-rustls"] } duckdb = { git = "https://github.com/frectonz/duckdb-rs.git", features = ["bundled"] } humantime = "2.1.0" +clickhouse = "0.11.6" [profile.release] strip = true diff --git a/flake.nix b/flake.nix index 13371f0..38312f0 100644 --- a/flake.nix +++ b/flake.nix @@ -90,6 +90,9 @@ pkgs.httpie pkgs.sqlite + + pkgs.openssl + pkgs.pkg-config ]; }; diff --git a/src/main.rs b/src/main.rs index a7fa391..0e0a55a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -59,6 +59,25 @@ enum Command { /// Path to the the duckdb file. database: String, }, + + /// A ClickHouse database. + Clickhouse { + /// Address to the clickhouse server. + #[arg(default_value = "http://localhost:8123")] + url: String, + + /// User we want to authentticate as. + #[arg(default_value = "default")] + user: String, + + /// Password we want to authentticate with. + #[arg(default_value = "")] + password: String, + + /// Name of the database. + #[arg(default_value = "default")] + database: String, + }, } #[tokio::main] @@ -88,6 +107,14 @@ async fn main() -> color_eyre::Result<()> { Command::Duckdb { database } => { AllDbs::Duckdb(duckdb::Db::open(database, args.timeout.into()).await?) } + Command::Clickhouse { + url, + user, + password, + database, + } => AllDbs::Clickhouse(Box::new( + clickhouse::Db::open(url, user, password, database, args.timeout.into()).await?, + )), }; let mut index_html = statics::get_index_html()?; @@ -211,6 +238,7 @@ enum AllDbs { Postgres(postgres::Db), Mysql(mysql::Db), Duckdb(duckdb::Db), + Clickhouse(Box), } #[async_trait] @@ -222,6 +250,7 @@ impl Database for AllDbs { AllDbs::Postgres(x) => x.overview().await, AllDbs::Mysql(x) => x.overview().await, AllDbs::Duckdb(x) => x.overview().await, + AllDbs::Clickhouse(x) => x.overview().await, } } @@ -232,6 +261,7 @@ impl Database for AllDbs { AllDbs::Postgres(x) => x.tables().await, AllDbs::Mysql(x) => x.tables().await, AllDbs::Duckdb(x) => x.tables().await, + AllDbs::Clickhouse(x) => x.tables().await, } } @@ -242,6 +272,7 @@ impl Database for AllDbs { AllDbs::Postgres(x) => x.table(name).await, AllDbs::Mysql(x) => x.table(name).await, AllDbs::Duckdb(x) => x.table(name).await, + AllDbs::Clickhouse(x) => x.table(name).await, } } @@ -256,6 +287,7 @@ impl Database for AllDbs { AllDbs::Postgres(x) => x.table_data(name, page).await, AllDbs::Mysql(x) => x.table_data(name, page).await, AllDbs::Duckdb(x) => x.table_data(name, page).await, + AllDbs::Clickhouse(x) => x.table_data(name, page).await, } } @@ -266,6 +298,7 @@ impl Database for AllDbs { AllDbs::Postgres(x) => x.query(query).await, AllDbs::Mysql(x) => x.query(query).await, AllDbs::Duckdb(x) => x.query(query).await, + AllDbs::Clickhouse(x) => x.query(query).await, } } } @@ -2370,6 +2403,363 @@ mod duckdb { } } +mod clickhouse { + use async_trait::async_trait; + use clickhouse::Client; + use color_eyre::eyre::OptionExt; + use std::time::Duration; + + use crate::{ + responses::{self, Count}, + Database, ROWS_PER_PAGE, + }; + + #[derive(Clone)] + pub struct Db { + conn: Client, + database: String, + _query_timeout: Duration, + } + + #[derive(serde::Deserialize, clickhouse::Row, Debug)] + pub struct ClickhouseCount { + pub name: String, + pub count: i64, + } + + impl Db { + pub async fn open( + url: String, + user: String, + password: String, + database: String, + query_timeout: Duration, + ) -> color_eyre::Result { + let conn = Client::default() + .with_url(url) + .with_user(user) + .with_password(password) + .with_database(&database); + + let tables: i32 = conn + .query( + r#" + SELECT count(*) + FROM system.tables + WHERE database = currentDatabase() + "#, + ) + .fetch_one() + .await?; + + tracing::info!( + "found {tables} table{} in {database}", + if tables == 1 { "" } else { "s" } + ); + + Ok(Self { + conn, + database, + _query_timeout: query_timeout, + }) + } + } + + #[async_trait] + impl Database for Db { + async fn overview(&self) -> color_eyre::Result { + let file_name = self.database.to_owned(); + + let db_size = String::new(); + let modified = None; + let created = None; + + let tables: i32 = self + .conn + .query( + r#" + SELECT count(*) + FROM system.tables + WHERE database = currentDatabase() + "#, + ) + .fetch_one() + .await?; + + let indexes: i32 = self + .conn + .query( + r#" + SELECT count(*) + FROM system.columns + WHERE database = currentDatabase() + AND (is_in_primary_key = true OR is_in_sorting_key = true) + "#, + ) + .fetch_one() + .await?; + + let triggers: i32 = 0; + + let views: i32 = self + .conn + .query( + r#" + SELECT count(*) + FROM system.tables + WHERE database = currentDatabase() + AND engine = 'View' + "#, + ) + .fetch_one() + .await?; + + let mut row_counts = self + .conn + .query( + r#" + SELECT name + FROM system.tables + WHERE database = currentDatabase() + "#, + ) + .fetch_all() + .await? + .into_iter() + .map(|name: String| Count { name, count: 0 }) + .collect::>(); + + for count in row_counts.iter_mut() { + count.count = self + .conn + .query(&format!("SELECT count(*) FROM {}", count.name)) + .fetch_one() + .await?; + } + + row_counts.sort_by(|a, b| b.count.cmp(&a.count)); + + let mut column_counts = self + .conn + .query( + r#" + SELECT table AS name, count() AS count + FROM system.columns + WHERE database = currentDatabase() + GROUP BY table + "#, + ) + .fetch_all::() + .await?; + + column_counts.sort_by(|a, b| b.count.cmp(&a.count)); + + let mut index_counts = self + .conn + .query( + r#" + SELECT name + FROM system.tables + WHERE database = currentDatabase() + "#, + ) + .fetch_all() + .await? + .into_iter() + .map(|name: String| Count { name, count: 0 }) + .collect::>(); + + for count in index_counts.iter_mut() { + count.count = self + .conn + .query( + r#" + SELECT count(*) + FROM system.columns + WHERE database = currentDatabase() + AND table = ? + AND (is_in_primary_key = true OR is_in_sorting_key = true) + "#, + ) + .bind(&count.name) + .fetch_one() + .await?; + } + + index_counts.sort_by(|a, b| b.count.cmp(&a.count)); + + Ok(responses::Overview { + file_name, + sqlite_version: None, + db_size, + created, + modified, + tables, + indexes, + triggers, + views, + row_counts, + column_counts: column_counts + .into_iter() + .map(|ClickhouseCount { name, count }| Count { + name, + count: count as i32, + }) + .collect(), + index_counts, + }) + } + + async fn tables(&self) -> color_eyre::Result { + let mut tables = self + .conn + .query( + r#" + SELECT name + FROM system.tables + WHERE database = currentDatabase() + "#, + ) + .fetch_all() + .await? + .into_iter() + .map(|name: String| Count { name, count: 0 }) + .collect::>(); + + for count in tables.iter_mut() { + count.count = self + .conn + .query(&format!("SELECT count(*) FROM {}", count.name)) + .fetch_one() + .await?; + } + + tables.sort_by_key(|r| r.count); + + Ok(responses::Tables { tables }) + } + + async fn table(&self, name: String) -> color_eyre::Result { + let sql: String = self + .conn + .query( + r#" + SELECT create_table_query + FROM system.tables + WHERE database = currentDatabase() + AND table = ? + "#, + ) + .bind(&name) + .fetch_one() + .await?; + + let row_count = self + .conn + .query(&format!("SELECT count(*) FROM {name}")) + .fetch_one() + .await?; + + let table_size = self + .conn + .query( + r#" + SELECT + formatReadableSize(sum(bytes)) as size + FROM system.parts WHERE table = ? + "#, + ) + .bind(&name) + .fetch_one::() + .await?; + + let index_count: i32 = self + .conn + .query( + r#" + SELECT count(*) + FROM system.columns + WHERE database = currentDatabase() + AND table = ? + AND (is_in_primary_key = true OR is_in_sorting_key = true) + "#, + ) + .bind(&name) + .fetch_one() + .await?; + + let column_count: i32 = self + .conn + .query( + r#" + SELECT count() AS count + FROM system.columns + WHERE database = currentDatabase() + AND table = ? + "#, + ) + .bind(&name) + .fetch_one() + .await?; + + Ok(responses::Table { + name, + sql: Some(sql), + row_count, + table_size, + index_count, + column_count, + }) + } + + async fn table_data( + &self, + name: String, + page: i32, + ) -> color_eyre::Result { + let mut columns = self + .conn + .query( + r#" + SELECT name + FROM system.columns + WHERE database = currentDatabase() + AND table = ? + "#, + ) + .bind(&name) + .fetch_all::() + .await?; + columns.truncate(5); + + let first_column = columns.first().ok_or_eyre("no first column found")?; + + let offset = (page - 1) * ROWS_PER_PAGE; + let _sql = format!( + r#" + SELECT {} FROM {name} + ORDER BY {first_column} + LIMIT {ROWS_PER_PAGE} + OFFSET {offset} + "#, + columns.join(",") + ); + + Ok(responses::TableData { + columns, + rows: Vec::new(), + }) + } + + async fn query(&self, _query: String) -> color_eyre::Result { + Ok(responses::Query { + columns: Vec::new(), + rows: Vec::new(), + }) + } + } +} + mod helpers { use duckdb::types::ValueRef as DuckdbValue; use libsql::Value as LibsqlValue; @@ -2434,7 +2824,7 @@ mod helpers { mod responses { use chrono::{DateTime, Utc}; - use serde::Serialize; + use serde::{Deserialize, Serialize}; #[derive(Serialize)] pub struct Overview { @@ -2452,7 +2842,7 @@ mod responses { pub index_counts: Vec, } - #[derive(Serialize)] + #[derive(Serialize, Deserialize, clickhouse::Row, Debug)] pub struct Count { pub name: String, pub count: i32,