From 1b292b9c584484bcffd579fe9abb3c947864eaef Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Dec 2024 17:15:44 +0800 Subject: [PATCH 01/16] feat: simple version switch --- tests/runner/src/env.rs | 15 +++++++ tests/runner/src/main.rs | 1 + .../standalone/test_simple.result | 43 +++++++++++++++++++ .../upgrade-compat/standalone/test_simple.sql | 20 +++++++++ 4 files changed, 79 insertions(+) create mode 100644 tests/upgrade-compat/standalone/test_simple.result create mode 100644 tests/upgrade-compat/standalone/test_simple.sql diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index bb5d74a26702..55de3de68810 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -45,6 +45,7 @@ use tokio::sync::Mutex as TokioMutex; use tokio_postgres::{Client as PgClient, SimpleQueryMessage as PgRow}; use crate::protocol_interceptor::{MYSQL, PROTOCOL_KEY}; +use crate::util::get_workspace_root; use crate::{util, ServerAddr}; const METASRV_ADDR: &str = "127.0.0.1:29302"; @@ -693,8 +694,22 @@ impl GreptimeDB { impl Database for GreptimeDB { async fn query(&self, ctx: QueryContext, query: String) -> Box { if ctx.context.contains_key("restart") && self.env.server_addrs.server_addr.is_none() { + self.env.restart_server(self).await; + } else if let Some(version) = ctx.context.get("version") { + if version == "latest" { + // use default latest by building db now + *self.env.bins_dir.lock().unwrap() = Some(util::get_binary_dir("debug")); + } else { + // use version in dir files + let root = get_workspace_root(); + let new_path = PathBuf::from_iter([&root, version]); + println!("DEBUG: {:?}", new_path); + *self.env.bins_dir.lock().unwrap() = Some(new_path); + } + self.env.restart_server(self).await; } + if let Some(protocol) = ctx.context.get(PROTOCOL_KEY) { // protocol is bound to be either "mysql" or "postgres" if protocol == MYSQL { diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index eca72f280e2a..579809b15010 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -141,6 +141,7 @@ async fn main() { args.bins_dir, ), ); + println!("Working directory: {:?}", std::env::current_dir()); runner.run().await.unwrap(); // clean up and exit diff --git a/tests/upgrade-compat/standalone/test_simple.result b/tests/upgrade-compat/standalone/test_simple.result new file mode 100644 index 000000000000..653364134ef4 --- /dev/null +++ b/tests/upgrade-compat/standalone/test_simple.result @@ -0,0 +1,43 @@ +-- SQLNESS ARG version=v0.9.5 +CREATE TABLE system_metrics ( + host STRING, + idc STRING, + cpu_util DOUBLE, + memory_util DOUBLE, + disk_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host, idc), + TIME INDEX(ts) +); + +Affected Rows: 0 + +INSERT INTO system_metrics +VALUES + ("host1", "idc_a", 11.8, 10.3, 10.3, 1667446797450), + ("host2", "idc_a", 80.0, 70.3, 90.0, 1667446797450), + ("host1", "idc_b", 50.0, 66.7, 40.6, 1667446797450); + +Affected Rows: 3 + +-- SQLNESS ARG version=latest +SHOW CREATE TABLE system_metrics; + ++----------------+-----------------------------------------------------------+ +| Table | Create Table | ++----------------+-----------------------------------------------------------+ +| system_metrics | CREATE TABLE IF NOT EXISTS "system_metrics" ( | +| | "host" STRING NULL, | +| | "idc" STRING NULL, | +| | "cpu_util" DOUBLE NULL, | +| | "memory_util" DOUBLE NULL, | +| | "disk_util" DOUBLE NULL, | +| | "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host", "idc") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++----------------+-----------------------------------------------------------+ + diff --git a/tests/upgrade-compat/standalone/test_simple.sql b/tests/upgrade-compat/standalone/test_simple.sql new file mode 100644 index 000000000000..3a2a06ea3f03 --- /dev/null +++ b/tests/upgrade-compat/standalone/test_simple.sql @@ -0,0 +1,20 @@ +-- SQLNESS ARG version=v0.9.5 +CREATE TABLE system_metrics ( + host STRING, + idc STRING, + cpu_util DOUBLE, + memory_util DOUBLE, + disk_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host, idc), + TIME INDEX(ts) +); + +INSERT INTO system_metrics +VALUES + ("host1", "idc_a", 11.8, 10.3, 10.3, 1667446797450), + ("host2", "idc_a", 80.0, 70.3, 90.0, 1667446797450), + ("host1", "idc_b", 50.0, 66.7, 40.6, 1667446797450); + +-- SQLNESS ARG version=latest +SHOW CREATE TABLE system_metrics; \ No newline at end of file From fe7e49cb798859e7b84b1f37d7fea7fb35932f33 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Dec 2024 17:24:58 +0800 Subject: [PATCH 02/16] chore: remove debug print --- tests/runner/src/env.rs | 1 - tests/runner/src/main.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index 55de3de68810..fd447093e445 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -703,7 +703,6 @@ impl Database for GreptimeDB { // use version in dir files let root = get_workspace_root(); let new_path = PathBuf::from_iter([&root, version]); - println!("DEBUG: {:?}", new_path); *self.env.bins_dir.lock().unwrap() = Some(new_path); } diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index 579809b15010..eca72f280e2a 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -141,7 +141,6 @@ async fn main() { args.bins_dir, ), ); - println!("Working directory: {:?}", std::env::current_dir()); runner.run().await.unwrap(); // clean up and exit From 508cccebe3880cb89eba876c18846452e46ffcf2 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Dec 2024 17:39:07 +0800 Subject: [PATCH 03/16] chore: add common folder --- tests/upgrade-compat/distributed/common | 1 + tests/upgrade-compat/standalone/{ => common}/test_simple.result | 0 tests/upgrade-compat/standalone/{ => common}/test_simple.sql | 0 3 files changed, 1 insertion(+) create mode 120000 tests/upgrade-compat/distributed/common rename tests/upgrade-compat/standalone/{ => common}/test_simple.result (100%) rename tests/upgrade-compat/standalone/{ => common}/test_simple.sql (100%) diff --git a/tests/upgrade-compat/distributed/common b/tests/upgrade-compat/distributed/common new file mode 120000 index 000000000000..2b0920287dc9 --- /dev/null +++ b/tests/upgrade-compat/distributed/common @@ -0,0 +1 @@ +../standalone/common \ No newline at end of file diff --git a/tests/upgrade-compat/standalone/test_simple.result b/tests/upgrade-compat/standalone/common/test_simple.result similarity index 100% rename from tests/upgrade-compat/standalone/test_simple.result rename to tests/upgrade-compat/standalone/common/test_simple.result diff --git a/tests/upgrade-compat/standalone/test_simple.sql b/tests/upgrade-compat/standalone/common/test_simple.sql similarity index 100% rename from tests/upgrade-compat/standalone/test_simple.sql rename to tests/upgrade-compat/standalone/common/test_simple.sql From 6982a218edcfe9c64fa1c921628d885f796a6b41 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Dec 2024 17:43:54 +0800 Subject: [PATCH 04/16] tests: add drop table --- tests/upgrade-compat/standalone/common/test_simple.result | 4 ++++ tests/upgrade-compat/standalone/common/test_simple.sql | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/upgrade-compat/standalone/common/test_simple.result b/tests/upgrade-compat/standalone/common/test_simple.result index 653364134ef4..ff2c340598a1 100644 --- a/tests/upgrade-compat/standalone/common/test_simple.result +++ b/tests/upgrade-compat/standalone/common/test_simple.result @@ -41,3 +41,7 @@ SHOW CREATE TABLE system_metrics; | | | +----------------+-----------------------------------------------------------+ +DROP TABLE system_metrics; + +Affected Rows: 0 + diff --git a/tests/upgrade-compat/standalone/common/test_simple.sql b/tests/upgrade-compat/standalone/common/test_simple.sql index 3a2a06ea3f03..84cc52464349 100644 --- a/tests/upgrade-compat/standalone/common/test_simple.sql +++ b/tests/upgrade-compat/standalone/common/test_simple.sql @@ -17,4 +17,6 @@ VALUES ("host1", "idc_b", 50.0, 66.7, 40.6, 1667446797450); -- SQLNESS ARG version=latest -SHOW CREATE TABLE system_metrics; \ No newline at end of file +SHOW CREATE TABLE system_metrics; + +DROP TABLE system_metrics; \ No newline at end of file From 8079e8fd063ad8a2bfc8b89f153dcf6eb53ac364 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Dec 2024 19:51:52 +0800 Subject: [PATCH 05/16] feat: pull versioned binary --- Cargo.lock | 187 +++++++++++++++++++++++++++++++++++++-- tests/runner/Cargo.toml | 6 ++ tests/runner/src/env.rs | 7 +- tests/runner/src/main.rs | 5 ++ tests/runner/src/util.rs | 144 +++++++++++++++++++++++++++++- 5 files changed, 339 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 920393daa030..d194027eee8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4149,6 +4149,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -4630,6 +4645,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap 2.6.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h3o" version = "0.6.4" @@ -5089,7 +5123,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -5112,6 +5146,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2 0.4.7", "http 1.1.0", "http-body 1.0.1", "httparse", @@ -5186,6 +5221,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.4.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.9" @@ -6537,7 +6588,7 @@ dependencies = [ "derive_builder 0.12.0", "etcd-client", "futures", - "h2", + "h2 0.3.26", "http-body 0.4.6", "humantime", "humantime-serde", @@ -7102,6 +7153,23 @@ dependencies = [ "winapi", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndk-context" version = "0.1.1" @@ -7521,12 +7589,50 @@ dependencies = [ "tokio-rustls 0.26.0", ] +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "opentelemetry" version = "0.21.0" @@ -9503,25 +9609,29 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.8" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-core", "futures-util", + "h2 0.4.7", "http 1.1.0", "http-body 1.0.1", "http-body-util", "hyper 1.4.1", "hyper-rustls", + "hyper-tls", "hyper-util", "ipnet", "js-sys", "log", "mime", "mime_guess", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -9534,7 +9644,9 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", + "system-configuration 0.6.1", "tokio", + "tokio-native-tls", "tokio-rustls 0.26.0", "tokio-util", "tower-service", @@ -10291,7 +10403,7 @@ dependencies = [ "sha2", "sha3", "socket2 0.4.10", - "system-configuration", + "system-configuration 0.5.1", "termios", "ucd", "unic-char-property", @@ -11400,14 +11512,20 @@ dependencies = [ "common-recordbatch", "common-time", "datatypes", + "flate2", + "hex", "mysql", + "reqwest", "serde", "serde_json", + "sha2", "sqlness", + "tar", "tempfile", "tinytemplate", "tokio", "tokio-postgres", + "tokio-stream", ] [[package]] @@ -11965,7 +12083,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -11978,6 +12107,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "table" version = "0.11.0" @@ -12181,6 +12320,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -12638,6 +12788,16 @@ dependencies = [ "tokio-metrics", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.12" @@ -12826,7 +12986,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", @@ -12854,7 +13014,7 @@ dependencies = [ "base64 0.21.7", "bytes", "flate2", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", @@ -14339,6 +14499,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "xml-rs" version = "0.8.22" diff --git a/tests/runner/Cargo.toml b/tests/runner/Cargo.toml index 71312c39dea3..89698714f4d1 100644 --- a/tests/runner/Cargo.toml +++ b/tests/runner/Cargo.toml @@ -21,7 +21,13 @@ serde.workspace = true serde_json.workspace = true tokio-postgres = { workspace = true } # sqlness 0.6.0 have a bug causing `cargo sqlness` to fail(see https://github.com/CeresDB/sqlness/issues/68) which is fixed in 0.6.1 +flate2 = "1.0" +hex = "0.4.3" +reqwest = "0.12.9" +sha2 = "=0.10.8" sqlness = "0.6.1" +tar = "0.4.43" tempfile.workspace = true tinytemplate = "1.2" tokio.workspace = true +tokio-stream.workspace = true diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index fd447093e445..24d9bdd9089d 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -45,7 +45,7 @@ use tokio::sync::Mutex as TokioMutex; use tokio_postgres::{Client as PgClient, SimpleQueryMessage as PgRow}; use crate::protocol_interceptor::{MYSQL, PROTOCOL_KEY}; -use crate::util::get_workspace_root; +use crate::util::{get_workspace_root, maybe_pull_binary}; use crate::{util, ServerAddr}; const METASRV_ADDR: &str = "127.0.0.1:29302"; @@ -75,6 +75,8 @@ pub struct Env { /// When running in CI, this is expected to be set. /// If not set, this runner will build the GreptimeDB binary itself when needed, and set this field by then. bins_dir: Arc>>, + /// Pull different versions of GreptimeDB on need. + pull_version_on_need: bool, } #[async_trait] @@ -101,12 +103,14 @@ impl Env { data_home: PathBuf, server_addrs: ServerAddr, wal: WalConfig, + pull_version_on_need: bool, bins_dir: Option, ) -> Self { Self { sqlness_home: data_home, server_addrs, wal, + pull_version_on_need, bins_dir: Arc::new(Mutex::new(bins_dir)), } } @@ -701,6 +705,7 @@ impl Database for GreptimeDB { *self.env.bins_dir.lock().unwrap() = Some(util::get_binary_dir("debug")); } else { // use version in dir files + maybe_pull_binary(version, self.env.pull_version_on_need).await; let root = get_workspace_root(); let new_path = PathBuf::from_iter([&root, version]); *self.env.bins_dir.lock().unwrap() = Some(new_path); diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index eca72f280e2a..a5386a3f05df 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -92,6 +92,10 @@ struct Args { /// This may affect future test runs. #[clap(long)] preserve_state: bool, + + /// Pull Different versions of GreptimeDB on need. + #[clap(long, default_value = "true")] + pull_version_on_need: bool, } #[tokio::main] @@ -138,6 +142,7 @@ async fn main() { sqlness_home.clone(), args.server_addr.clone(), wal, + args.pull_version_on_need, args.bins_dir, ), ); diff --git a/tests/runner/src/util.rs b/tests/runner/src/util.rs index 04c336e1485c..574738ee4310 100644 --- a/tests/runner/src/util.rs +++ b/tests/runner/src/util.rs @@ -12,18 +12,160 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::io::Read; use std::net::SocketAddr; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Duration; +use sha2::{Digest, Sha256}; use tokio::io::AsyncWriteExt; use tokio::net::TcpSocket; use tokio::time; +use tokio_stream::StreamExt; /// Check port every 0.1 second. const PORT_CHECK_INTERVAL: Duration = Duration::from_millis(100); +fn http_proxy() -> Option { + for proxy in ["http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY"] { + if let Ok(proxy_addr) = std::env::var(proxy) { + println!("Setting Proxy from env: {}={}", proxy, proxy_addr); + return Some(proxy_addr); + } + } + None +} + +fn https_proxy() -> Option { + for proxy in ["https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"] { + if let Ok(proxy_addr) = std::env::var(proxy) { + println!("Setting Proxy from env: {}={}", proxy, proxy_addr); + return Some(proxy_addr); + } + } + None +} + +async fn download_files(url: &str, path: &str) { + let proxy = if url.starts_with("http://") { + http_proxy().map(|proxy| reqwest::Proxy::http(proxy).unwrap()) + } else if url.starts_with("https://") { + https_proxy().map(|proxy| reqwest::Proxy::https(proxy).unwrap()) + } else { + None + }; + + let client = proxy + .map(|proxy| reqwest::Client::builder().proxy(proxy).build().unwrap()) + .unwrap_or(reqwest::Client::new()); + + let mut file = tokio::fs::File::create(path).await.unwrap(); + println!("Downloading {}...", url); + + let mut stream = client.get(url).send().await.unwrap().bytes_stream(); + let mut size_downloaded = 0; + + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result.unwrap(); + size_downloaded += chunk.len(); + print!("\rDownloaded {} bytes", size_downloaded); + file.write_all(&chunk).await.unwrap(); + } + + file.flush().await.unwrap(); + + println!("\nDownloaded {}", url); +} + +fn decompress(archive: &str, dest: &str) { + let tar = std::fs::File::open(archive).unwrap(); + let dec = flate2::read::GzDecoder::new(tar); + let mut a = tar::Archive::new(dec); + a.unpack(dest).unwrap(); +} + +/// Use curl to download the binary from the release page. +/// +/// # Arguments +/// +/// * `version` - The version of the binary to download. i.e. "v0.9.5" +pub async fn pull_binary(version: &str) { + let os = std::env::consts::OS; + let arch = match std::env::consts::ARCH { + "x86_64" => "amd64", + "aarch64" => "arm64", + _ => panic!("Unsupported arch: {}", std::env::consts::ARCH), + }; + let triple = format!("greptime-{}-{}-{}", os, arch, version); + let filename = format!("{triple}.tar.gz"); + // TODO: make it cross platform and aware of proxy + let url = format!( + "https://github.com/GreptimeTeam/greptimedb/releases/download/{version}/{filename}" + ); + println!("Downloading {version} binary from {}", url); + // mkdir {version} + + std::fs::create_dir(version).unwrap(); + + let archive = Path::new(version).join(filename); + let folder_path = Path::new(version); + + // download the binary to the version directory + download_files(&url, &archive.to_string_lossy()).await; + + let checksum_file = format!("{triple}.sha256sum"); + let checksum_url = format!( + "https://github.com/GreptimeTeam/greptimedb/releases/download/{version}/{checksum_file}" + ); + download_files( + &checksum_url, + &PathBuf::from_iter([version, &checksum_file]).to_string_lossy(), + ) + .await; + + // verify the checksum + let mut file = std::fs::File::open(&archive).unwrap(); + let mut sha256 = Sha256::new(); + std::io::copy(&mut file, &mut sha256).unwrap(); + let checksum: Vec = sha256.finalize().to_vec(); + + let mut expected_checksum = + std::fs::File::open(PathBuf::from_iter([version, &checksum_file])).unwrap(); + let mut buf = String::new(); + expected_checksum.read_to_string(&mut buf).unwrap(); + let expected_checksum = hex::decode(buf.lines().next().unwrap()).unwrap(); + + assert_eq!( + checksum, expected_checksum, + "Downloaded file is corrupted, checksum mismatched" + ); + + decompress(&archive.to_string_lossy(), &folder_path.to_string_lossy()); + println!("Downloaded and extracted {version} binary to {folder_path:?}"); + + // move the binary to the version directory + std::fs::rename( + PathBuf::from_iter([version, &triple, "greptime"]), + PathBuf::from_iter([version, "greptime"]), + ) + .unwrap(); + + // remove the archive and inner folder + std::fs::remove_file(&archive).unwrap(); + std::fs::remove_dir(PathBuf::from_iter([version, &triple])).unwrap(); +} + +/// Pull the binary if it does not exist and `pull_version_on_need` is true. +pub async fn maybe_pull_binary(version: &str, pull_version_on_need: bool) { + let exist = Path::new(version).is_dir(); + match (exist, pull_version_on_need){ + (true, _) => println!("Binary {version} exists"), + (false, false) => panic!("Binary {version} does not exist, please run with --pull-version-on-need or manually download it"), + (false, true) => { pull_binary(version).await; }, + } +} + /// Get the dir of test cases. This function only works when the runner is run /// under the project's dir because it depends on some envs set by cargo. pub fn get_case_dir(case_dir: Option) -> String { From 128b45e131d44695498fcdb82d362138bd75ccb9 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Dec 2024 20:09:22 +0800 Subject: [PATCH 06/16] chore: don't use native-tls --- Cargo.lock | 155 ++-------------------------------------- tests/runner/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 150 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d194027eee8c..2ee842146f52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4149,21 +4149,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -4645,25 +4630,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "h2" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http 1.1.0", - "indexmap 2.6.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h3o" version = "0.6.4" @@ -5123,7 +5089,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -5146,7 +5112,6 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.7", "http 1.1.0", "http-body 1.0.1", "httparse", @@ -5221,22 +5186,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.4.1", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.9" @@ -6588,7 +6537,7 @@ dependencies = [ "derive_builder 0.12.0", "etcd-client", "futures", - "h2 0.3.26", + "h2", "http-body 0.4.6", "humantime", "humantime-serde", @@ -7153,23 +7102,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "native-tls" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "ndk-context" version = "0.1.1" @@ -7589,50 +7521,12 @@ dependencies = [ "tokio-rustls 0.26.0", ] -[[package]] -name = "openssl" -version = "0.10.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" -dependencies = [ - "bitflags 2.6.0", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.90", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "opentelemetry" version = "0.21.0" @@ -9615,23 +9509,19 @@ checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", - "encoding_rs", "futures-core", "futures-util", - "h2 0.4.7", "http 1.1.0", "http-body 1.0.1", "http-body-util", "hyper 1.4.1", "hyper-rustls", - "hyper-tls", "hyper-util", "ipnet", "js-sys", "log", "mime", "mime_guess", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -9644,9 +9534,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", - "system-configuration 0.6.1", "tokio", - "tokio-native-tls", "tokio-rustls 0.26.0", "tokio-util", "tower-service", @@ -10403,7 +10291,7 @@ dependencies = [ "sha2", "sha3", "socket2 0.4.10", - "system-configuration 0.5.1", + "system-configuration", "termios", "ucd", "unic-char-property", @@ -12083,18 +11971,7 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys 0.5.0", -] - -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags 2.6.0", - "core-foundation", - "system-configuration-sys 0.6.0", + "system-configuration-sys", ] [[package]] @@ -12107,16 +11984,6 @@ dependencies = [ "libc", ] -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "table" version = "0.11.0" @@ -12788,16 +12655,6 @@ dependencies = [ "tokio-metrics", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-postgres" version = "0.7.12" @@ -12986,7 +12843,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", @@ -13014,7 +12871,7 @@ dependencies = [ "base64 0.21.7", "bytes", "flate2", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", diff --git a/tests/runner/Cargo.toml b/tests/runner/Cargo.toml index 89698714f4d1..148a4eba70f5 100644 --- a/tests/runner/Cargo.toml +++ b/tests/runner/Cargo.toml @@ -23,7 +23,7 @@ tokio-postgres = { workspace = true } # sqlness 0.6.0 have a bug causing `cargo sqlness` to fail(see https://github.com/CeresDB/sqlness/issues/68) which is fixed in 0.6.1 flate2 = "1.0" hex = "0.4.3" -reqwest = "0.12.9" +reqwest = { version = "0.12.9", default-features = false, features = ["rustls-tls"] } sha2 = "=0.10.8" sqlness = "0.6.1" tar = "0.4.43" From 87c76f76e501e0e05466d5c1ed12c617f1a29182 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Dec 2024 20:25:51 +0800 Subject: [PATCH 07/16] chore: rm outdated docs --- tests/runner/src/util.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/runner/src/util.rs b/tests/runner/src/util.rs index 574738ee4310..5f9cbfa1bc59 100644 --- a/tests/runner/src/util.rs +++ b/tests/runner/src/util.rs @@ -99,13 +99,13 @@ pub async fn pull_binary(version: &str) { }; let triple = format!("greptime-{}-{}-{}", os, arch, version); let filename = format!("{triple}.tar.gz"); - // TODO: make it cross platform and aware of proxy + let url = format!( "https://github.com/GreptimeTeam/greptimedb/releases/download/{version}/{filename}" ); println!("Downloading {version} binary from {}", url); - // mkdir {version} + // mkdir {version} std::fs::create_dir(version).unwrap(); let archive = Path::new(version).join(filename); @@ -138,7 +138,7 @@ pub async fn pull_binary(version: &str) { assert_eq!( checksum, expected_checksum, - "Downloaded file is corrupted, checksum mismatched" + "Checksum mismatched, downloaded file is corrupted" ); decompress(&archive.to_string_lossy(), &folder_path.to_string_lossy()); From f407ecb8957f9b78bfb11a2a910764506574e2dc Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Dec 2024 20:36:30 +0800 Subject: [PATCH 08/16] chore: new line --- tests/upgrade-compat/standalone/common/test_simple.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/upgrade-compat/standalone/common/test_simple.sql b/tests/upgrade-compat/standalone/common/test_simple.sql index 84cc52464349..0f8daa0985ab 100644 --- a/tests/upgrade-compat/standalone/common/test_simple.sql +++ b/tests/upgrade-compat/standalone/common/test_simple.sql @@ -19,4 +19,4 @@ VALUES -- SQLNESS ARG version=latest SHOW CREATE TABLE system_metrics; -DROP TABLE system_metrics; \ No newline at end of file +DROP TABLE system_metrics; From a7d531b95f6eb92424b3d93932dd5e1b6dae417f Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Dec 2024 20:51:46 +0800 Subject: [PATCH 09/16] fix: save old bin dir --- tests/runner/src/env.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index 24d9bdd9089d..0e80967c8d04 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -77,6 +77,8 @@ pub struct Env { bins_dir: Arc>>, /// Pull different versions of GreptimeDB on need. pull_version_on_need: bool, + /// old bins dir, useful when switching versions + old_bins_dir: Arc>>, } #[async_trait] @@ -112,6 +114,7 @@ impl Env { wal, pull_version_on_need, bins_dir: Arc::new(Mutex::new(bins_dir)), + old_bins_dir: Arc::new(Mutex::new(None)), } } @@ -700,9 +703,13 @@ impl Database for GreptimeDB { if ctx.context.contains_key("restart") && self.env.server_addrs.server_addr.is_none() { self.env.restart_server(self).await; } else if let Some(version) = ctx.context.get("version") { + if self.env.old_bins_dir.lock().unwrap().is_none() { + // save old bins dir + *self.env.old_bins_dir.lock().unwrap() = self.env.bins_dir.lock().unwrap().clone(); + } if version == "latest" { // use default latest by building db now - *self.env.bins_dir.lock().unwrap() = Some(util::get_binary_dir("debug")); + *self.env.bins_dir.lock().unwrap() = self.env.old_bins_dir.lock().unwrap().clone(); } else { // use version in dir files maybe_pull_binary(version, self.env.pull_version_on_need).await; From 462dd4e67ac1e54fe29f2224709be5379e456423 Mon Sep 17 00:00:00 2001 From: discord9 Date: Tue, 10 Dec 2024 13:08:29 +0800 Subject: [PATCH 10/16] fix: switch version restart all node --- tests/runner/src/env.rs | 83 ++++++++-- .../standalone/common/test_ttl.result | 153 ++++++++++++++++++ .../standalone/common/test_ttl.sql | 42 +++++ 3 files changed, 263 insertions(+), 15 deletions(-) create mode 100644 tests/upgrade-compat/standalone/common/test_ttl.result create mode 100644 tests/upgrade-compat/standalone/common/test_ttl.sql diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index 0e80967c8d04..5175c785b085 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -160,12 +160,12 @@ impl Env { let mut greptimedb = self.connect_db(&Default::default()).await; - greptimedb.metasrv_process = Some(meta_server); + greptimedb.metasrv_process = Some(meta_server).into(); greptimedb.server_processes = Some(Arc::new(Mutex::new(vec![ datanode_1, datanode_2, datanode_3, ]))); - greptimedb.frontend_process = Some(frontend); - greptimedb.flownode_process = Some(flownode); + greptimedb.frontend_process = Some(frontend).into(); + greptimedb.flownode_process = Some(flownode).into(); greptimedb.is_standalone = false; greptimedb.ctx = db_ctx; @@ -245,9 +245,9 @@ impl Env { pg_client: TokioMutex::new(pg_client), mysql_client: TokioMutex::new(mysql_client), server_processes: None, - metasrv_process: None, - frontend_process: None, - flownode_process: None, + metasrv_process: None.into(), + frontend_process: None.into(), + flownode_process: None.into(), ctx: GreptimeDBContext { time: 0, datanode_id: Default::default(), @@ -460,7 +460,7 @@ impl Env { } /// stop and restart the server process - async fn restart_server(&self, db: &GreptimeDB) { + async fn restart_server(&self, db: &GreptimeDB, is_full_restart: bool) { { if let Some(server_process) = db.server_processes.clone() { let mut server_processes = server_process.lock().unwrap(); @@ -468,6 +468,23 @@ impl Env { Env::stop_server(server_process); } } + if is_full_restart { + if let Some(mut metasrv_process) = + db.metasrv_process.lock().expect("poisoned lock").take() + { + Env::stop_server(&mut metasrv_process); + } + if let Some(mut frontend_process) = + db.frontend_process.lock().expect("poisoned lock").take() + { + Env::stop_server(&mut frontend_process); + } + if let Some(mut flownode_process) = + db.flownode_process.lock().expect("poisoned lock").take() + { + Env::stop_server(&mut flownode_process); + } + } } // check if the server is distributed or standalone @@ -476,12 +493,33 @@ impl Env { vec![new_server_process] } else { db.ctx.reset_datanode_id(); + if is_full_restart { + let metasrv = self.start_server("metasrv", &db.ctx, false).await; + db.metasrv_process + .lock() + .expect("lock poisoned") + .replace(metasrv); + } let mut processes = vec![]; for _ in 0..3 { let new_server_process = self.start_server("datanode", &db.ctx, false).await; processes.push(new_server_process); } + + if is_full_restart { + let frontend = self.start_server("frontend", &db.ctx, false).await; + db.frontend_process + .lock() + .expect("lock poisoned") + .replace(frontend); + + let flownode = self.start_server("flownode", &db.ctx, false).await; + db.flownode_process + .lock() + .expect("lock poisoned") + .replace(flownode); + } processes }; @@ -588,9 +626,9 @@ impl Env { pub struct GreptimeDB { server_processes: Option>>>, - metasrv_process: Option, - frontend_process: Option, - flownode_process: Option, + metasrv_process: Mutex>, + frontend_process: Mutex>, + flownode_process: Mutex>, grpc_client: TokioMutex, pg_client: TokioMutex, mysql_client: TokioMutex, @@ -701,7 +739,7 @@ impl GreptimeDB { impl Database for GreptimeDB { async fn query(&self, ctx: QueryContext, query: String) -> Box { if ctx.context.contains_key("restart") && self.env.server_addrs.server_addr.is_none() { - self.env.restart_server(self).await; + self.env.restart_server(self, false).await; } else if let Some(version) = ctx.context.get("version") { if self.env.old_bins_dir.lock().unwrap().is_none() { // save old bins dir @@ -718,7 +756,7 @@ impl Database for GreptimeDB { *self.env.bins_dir.lock().unwrap() = Some(new_path); } - self.env.restart_server(self).await; + self.env.restart_server(self, true).await; } if let Some(protocol) = ctx.context.get(PROTOCOL_KEY) { @@ -746,15 +784,30 @@ impl GreptimeDB { ); } } - if let Some(mut metasrv) = self.metasrv_process.take() { + if let Some(mut metasrv) = self + .metasrv_process + .lock() + .expect("someone else panic when holding lock") + .take() + { Env::stop_server(&mut metasrv); println!("Metasrv (pid = {}) is stopped", metasrv.id()); } - if let Some(mut frontend) = self.frontend_process.take() { + if let Some(mut frontend) = self + .frontend_process + .lock() + .expect("someone else panic when holding lock") + .take() + { Env::stop_server(&mut frontend); println!("Frontend (pid = {}) is stopped", frontend.id()); } - if let Some(mut flownode) = self.flownode_process.take() { + if let Some(mut flownode) = self + .flownode_process + .lock() + .expect("someone else panic when holding lock") + .take() + { Env::stop_server(&mut flownode); println!("Flownode (pid = {}) is stopped", flownode.id()); } diff --git a/tests/upgrade-compat/standalone/common/test_ttl.result b/tests/upgrade-compat/standalone/common/test_ttl.result new file mode 100644 index 000000000000..d06bc629b668 --- /dev/null +++ b/tests/upgrade-compat/standalone/common/test_ttl.result @@ -0,0 +1,153 @@ +-- SQLNESS ARG version=v0.9.5 +CREATE TABLE test_ttl_0s(ts TIMESTAMP TIME INDEX, val INT) WITH (ttl = '0 second'); + +Affected Rows: 0 + +CREATE TABLE test_ttl_1s(ts TIMESTAMP TIME INDEX, val INT) WITH (ttl = '1 second'); + +Affected Rows: 0 + +CREATE TABLE test_ttl_none(ts TIMESTAMP TIME INDEX, val INT); + +Affected Rows: 0 + +CREATE DATABASE ttl_db_1s WITH (ttl = '1 second'); + +Affected Rows: 1 + +CREATE DATABASE ttl_db_0s WITH (ttl = '0 second'); + +Affected Rows: 1 + +CREATE DATABASE ttl_db_none; + +Affected Rows: 1 + +-- SQLNESS ARG version=latest +SHOW TABLES; + ++---------------+ +| Tables | ++---------------+ +| numbers | +| test_ttl_0s | +| test_ttl_1s | +| test_ttl_none | ++---------------+ + +SHOW CREATE TABLE test_ttl_1s; + ++-------------+--------------------------------------------+ +| Table | Create Table | ++-------------+--------------------------------------------+ +| test_ttl_1s | CREATE TABLE IF NOT EXISTS "test_ttl_1s" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "val" INT NULL, | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | ttl = '1s' | +| | ) | ++-------------+--------------------------------------------+ + +SHOW CREATE TABLE test_ttl_0s; + ++-------------+--------------------------------------------+ +| Table | Create Table | ++-------------+--------------------------------------------+ +| test_ttl_0s | CREATE TABLE IF NOT EXISTS "test_ttl_0s" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "val" INT NULL, | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | ttl = '0s' | +| | ) | ++-------------+--------------------------------------------+ + +SHOW CREATE TABLE test_ttl_none; + ++---------------+----------------------------------------------+ +| Table | Create Table | ++---------------+----------------------------------------------+ +| test_ttl_none | CREATE TABLE IF NOT EXISTS "test_ttl_none" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "val" INT NULL, | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++---------------+----------------------------------------------+ + +DROP TABLE test_ttl_1s; + +Affected Rows: 0 + +DROP TABLE test_ttl_0s; + +Affected Rows: 0 + +DROP TABLE test_ttl_none; + +Affected Rows: 0 + +SHOW DATABASES; + ++--------------------+ +| Database | ++--------------------+ +| greptime_private | +| information_schema | +| public | +| ttl_db_0s | +| ttl_db_1s | +| ttl_db_none | ++--------------------+ + +SHOW CREATE DATABASE ttl_db_1s; + ++-----------+-----------------------------------------+ +| Database | Create Database | ++-----------+-----------------------------------------+ +| ttl_db_1s | CREATE DATABASE IF NOT EXISTS ttl_db_1s | +| | WITH( | +| | ttl = '1s' | +| | ) | ++-----------+-----------------------------------------+ + +SHOW CREATE DATABASE ttl_db_0s; + ++-----------+-----------------------------------------+ +| Database | Create Database | ++-----------+-----------------------------------------+ +| ttl_db_0s | CREATE DATABASE IF NOT EXISTS ttl_db_0s | +| | WITH( | +| | ttl = '0s' | +| | ) | ++-----------+-----------------------------------------+ + +SHOW CREATE DATABASE ttl_db_none; + ++-------------+-------------------------------------------+ +| Database | Create Database | ++-------------+-------------------------------------------+ +| ttl_db_none | CREATE DATABASE IF NOT EXISTS ttl_db_none | ++-------------+-------------------------------------------+ + +DROP DATABASE ttl_db_1s; + +Affected Rows: 0 + +DROP DATABASE ttl_db_0s; + +Affected Rows: 0 + +DROP DATABASE ttl_db_none; + +Affected Rows: 0 + diff --git a/tests/upgrade-compat/standalone/common/test_ttl.sql b/tests/upgrade-compat/standalone/common/test_ttl.sql new file mode 100644 index 000000000000..3462fd22444f --- /dev/null +++ b/tests/upgrade-compat/standalone/common/test_ttl.sql @@ -0,0 +1,42 @@ + +-- SQLNESS ARG version=v0.9.5 +CREATE TABLE test_ttl_0s(ts TIMESTAMP TIME INDEX, val INT) WITH (ttl = '0 second'); + +CREATE TABLE test_ttl_1s(ts TIMESTAMP TIME INDEX, val INT) WITH (ttl = '1 second'); + +CREATE TABLE test_ttl_none(ts TIMESTAMP TIME INDEX, val INT); + +CREATE DATABASE ttl_db_1s WITH (ttl = '1 second'); + +CREATE DATABASE ttl_db_0s WITH (ttl = '0 second'); + +CREATE DATABASE ttl_db_none; + +-- SQLNESS ARG version=latest +SHOW TABLES; + +SHOW CREATE TABLE test_ttl_1s; + +SHOW CREATE TABLE test_ttl_0s; + +SHOW CREATE TABLE test_ttl_none; + +DROP TABLE test_ttl_1s; + +DROP TABLE test_ttl_0s; + +DROP TABLE test_ttl_none; + +SHOW DATABASES; + +SHOW CREATE DATABASE ttl_db_1s; + +SHOW CREATE DATABASE ttl_db_0s; + +SHOW CREATE DATABASE ttl_db_none; + +DROP DATABASE ttl_db_1s; + +DROP DATABASE ttl_db_0s; + +DROP DATABASE ttl_db_none; From 5f6a10cab469f6942f71ae5612a1bc411155ac65 Mon Sep 17 00:00:00 2001 From: discord9 Date: Tue, 10 Dec 2024 19:33:13 +0800 Subject: [PATCH 11/16] feat: use etcd --- tests/conf/metasrv-test.toml.template | 10 +++ tests/runner/src/env.rs | 35 ++++++++ tests/runner/src/main.rs | 23 +++++ tests/runner/src/util.rs | 116 ++++++++++++++++++++++++-- 4 files changed, 178 insertions(+), 6 deletions(-) diff --git a/tests/conf/metasrv-test.toml.template b/tests/conf/metasrv-test.toml.template index 8d27aad3c4b2..5519cc65e1a3 100644 --- a/tests/conf/metasrv-test.toml.template +++ b/tests/conf/metasrv-test.toml.template @@ -1,4 +1,14 @@ flush_stats_factor = 1 +{{ if setup_etcd }} +## Store server address default to etcd store. +store_addrs = [{store_addrs | unescaped}] + +## Store data in memory. +use_memory_store = false + +## The datastore for meta server. +backend = "EtcdStore" +{{ endif }} [wal] {{ if is_raft_engine }} provider = "raft_engine" diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index 5175c785b085..9212288cd7e7 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -65,6 +65,12 @@ pub enum WalConfig { }, } +#[derive(Clone)] +pub struct StoreConfig { + pub store_addrs: Vec, + pub setup_etcd: bool, +} + #[derive(Clone)] pub struct Env { sqlness_home: PathBuf, @@ -79,6 +85,8 @@ pub struct Env { pull_version_on_need: bool, /// old bins dir, useful when switching versions old_bins_dir: Arc>>, + /// Store address for metasrv metadata + store_config: StoreConfig, } #[async_trait] @@ -107,6 +115,7 @@ impl Env { wal: WalConfig, pull_version_on_need: bool, bins_dir: Option, + store_config: StoreConfig, ) -> Self { Self { sqlness_home: data_home, @@ -115,6 +124,7 @@ impl Env { pull_version_on_need, bins_dir: Arc::new(Mutex::new(bins_dir)), old_bins_dir: Arc::new(Mutex::new(None)), + store_config, } } @@ -144,6 +154,7 @@ impl Env { } else { self.build_db(); self.setup_wal(); + self.setup_etcd(); let db_ctx = GreptimeDBContext::new(self.wal.clone()); @@ -539,6 +550,19 @@ impl Env { } } + /// Setup etcd if needed. + fn setup_etcd(&self) { + if self.store_config.setup_etcd { + let client_ports = self + .store_config + .store_addrs + .iter() + .map(|s| s.split(':').nth(1).unwrap().parse::().unwrap()) + .collect::>(); + util::setup_etcd(client_ports, None, None); + } + } + /// Generate config file to `/tmp/{subcommand}-{current_time}.toml` fn generate_config_file(&self, subcommand: &str, db_ctx: &GreptimeDBContext) -> String { let mut tt = TinyTemplate::new(); @@ -555,6 +579,8 @@ impl Env { procedure_dir: String, is_raft_engine: bool, kafka_wal_broker_endpoints: String, + setup_etcd: bool, + store_addrs: String, } let data_home = self.sqlness_home.join(format!("greptimedb-{subcommand}")); @@ -568,6 +594,15 @@ impl Env { procedure_dir, is_raft_engine: db_ctx.is_raft_engine(), kafka_wal_broker_endpoints: db_ctx.kafka_wal_broker_endpoints(), + setup_etcd: self.store_config.setup_etcd, + store_addrs: self + .store_config + .store_addrs + .clone() + .iter() + .map(|p| format!("\"{p}\"")) + .collect::>() + .join(","), }; let rendered = tt.render(subcommand, &ctx).unwrap(); diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index a5386a3f05df..aed0de953531 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -22,6 +22,8 @@ use env::{Env, WalConfig}; use sqlness::interceptor::Registry; use sqlness::{ConfigBuilder, Runner}; +use crate::env::StoreConfig; + mod env; mod protocol_interceptor; mod util; @@ -96,6 +98,14 @@ struct Args { /// Pull Different versions of GreptimeDB on need. #[clap(long, default_value = "true")] pull_version_on_need: bool, + + /// The store addresses for the raft engine. + #[clap(long, default_value = "0.0.0.0:2379")] + store_addrs: Vec, + + /// Whether to setup etcd, by default it is true. + #[clap(long, default_value = "false")] + setup_etcd: bool, } #[tokio::main] @@ -114,6 +124,11 @@ async fn main() { Arc::new(protocol_interceptor::ProtocolInterceptorFactory), ); + if let Some(d) = &args.case_dir { + if !d.is_dir() { + panic!("{} is not a directory", d.display()); + } + } let config = ConfigBuilder::default() .case_dir(util::get_case_dir(args.case_dir)) .fail_fast(args.fail_fast) @@ -136,6 +151,11 @@ async fn main() { }, }; + let store = StoreConfig { + store_addrs: args.store_addrs.clone(), + setup_etcd: args.setup_etcd, + }; + let runner = Runner::new( config, Env::new( @@ -144,12 +164,15 @@ async fn main() { wal, args.pull_version_on_need, args.bins_dir, + store, ), ); runner.run().await.unwrap(); // clean up and exit if !args.preserve_state { + println!("Stopping etcd"); + util::stop_etcd(); println!("Removing state in {:?}", sqlness_home); tokio::fs::remove_dir_all(sqlness_home).await.unwrap(); } diff --git a/tests/runner/src/util.rs b/tests/runner/src/util.rs index 5f9cbfa1bc59..61bed6e233e0 100644 --- a/tests/runner/src/util.rs +++ b/tests/runner/src/util.rs @@ -30,7 +30,7 @@ const PORT_CHECK_INTERVAL: Duration = Duration::from_millis(100); fn http_proxy() -> Option { for proxy in ["http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY"] { if let Ok(proxy_addr) = std::env::var(proxy) { - println!("Setting Proxy from env: {}={}", proxy, proxy_addr); + println!("Getting Proxy from env var: {}={}", proxy, proxy_addr); return Some(proxy_addr); } } @@ -40,7 +40,7 @@ fn http_proxy() -> Option { fn https_proxy() -> Option { for proxy in ["https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"] { if let Ok(proxy_addr) = std::env::var(proxy) { - println!("Setting Proxy from env: {}={}", proxy, proxy_addr); + println!("Getting Proxy from env var: {}={}", proxy, proxy_addr); return Some(proxy_addr); } } @@ -57,19 +57,37 @@ async fn download_files(url: &str, path: &str) { }; let client = proxy - .map(|proxy| reqwest::Client::builder().proxy(proxy).build().unwrap()) + .map(|proxy| { + reqwest::Client::builder() + .proxy(proxy) + .build() + .expect("Failed to build client") + }) .unwrap_or(reqwest::Client::new()); - let mut file = tokio::fs::File::create(path).await.unwrap(); + let mut file = tokio::fs::File::create(path) + .await + .unwrap_or_else(|_| panic!("Failed to create file in {path}")); println!("Downloading {}...", url); - let mut stream = client.get(url).send().await.unwrap().bytes_stream(); + let resp = client + .get(url) + .send() + .await + .expect("Failed to send download request"); + let len = resp.content_length(); + let mut stream = resp.bytes_stream(); let mut size_downloaded = 0; while let Some(chunk_result) = stream.next().await { let chunk = chunk_result.unwrap(); size_downloaded += chunk.len(); - print!("\rDownloaded {} bytes", size_downloaded); + if let Some(len) = len { + print!("\rDownloading {}/{} bytes", size_downloaded, len); + } else { + print!("\rDownloaded {} bytes", size_downloaded); + } + file.write_all(&chunk).await.unwrap(); } @@ -166,6 +184,92 @@ pub async fn maybe_pull_binary(version: &str, pull_version_on_need: bool) { } } +/// Set up a standalone etcd in docker. +pub fn setup_etcd(client_ports: Vec, peer_port: Option, etcd_version: Option<&str>) { + if std::process::Command::new("docker") + .args(["-v"]) + .status() + .is_err() + { + panic!("Docker is not installed"); + } + let peer_port = peer_port.unwrap_or(2380); + let exposed_port: Vec<_> = client_ports.iter().chain(Some(&peer_port)).collect(); + let exposed_port_str = exposed_port + .iter() + .flat_map(|p| ["-p".to_string(), format!("{p}:{p}")]) + .collect::>(); + let etcd_version = etcd_version.unwrap_or("v3.5.17"); + let etcd_image = format!("quay.io/coreos/etcd:{etcd_version}"); + let peer_url = format!("http://0.0.0.0:{peer_port}"); + + let client_ports_fmt = client_ports + .iter() + .map(|p| format!("http://0.0.0.0:{p}")) + .collect::>(); + let mut arg_list = vec![]; + arg_list.extend([ + "run", + "-d", + "-v", + "/usr/share/ca-certificates/:/etc/ssl/certs", + ]); + arg_list.extend(exposed_port_str.iter().map(std::ops::Deref::deref)); + arg_list.extend([ + "--name", + "etcd", + &etcd_image, + "etcd", + "-name", + "etcd0", + "-listen-client-urls", + ]); + + arg_list.extend(client_ports_fmt.iter().map(std::ops::Deref::deref)); + arg_list.extend(["-listen-peer-urls", &peer_url, "-initial-cluster-state new"]); + + let mut cmd = std::process::Command::new("docker"); + + cmd.args(arg_list); + + println!("Starting etcd with command: {:?}", cmd); + + let status = cmd.status(); + if status.is_err() { + panic!("Failed to start etcd: {:?}", status); + } else if let Ok(status) = status { + if status.success() { + println!( + "Started etcd with client ports {:?} and peer port {}, statues:{status:?}", + client_ports, peer_port + ); + } else { + panic!("Failed to start etcd: {:?}", status); + } + } +} + +/// Stop the etcd container +pub fn stop_etcd() { + let status = std::process::Command::new("docker") + .args(["container", "stop", "etcd"]) + .status(); + if status.is_err() { + panic!("Failed to stop etcd: {:?}", status); + } else { + println!("Stopped etcd"); + } + // rm the container + let status = std::process::Command::new("docker") + .args(["container", "rm", "etcd"]) + .status(); + if status.is_err() { + panic!("Failed to remove etcd container: {:?}", status); + } else { + println!("Removed etcd container"); + } +} + /// Get the dir of test cases. This function only works when the runner is run /// under the project's dir because it depends on some envs set by cargo. pub fn get_case_dir(case_dir: Option) -> String { From 6fefa6c8ff49dac7f322c6ba61395c5d11fb9f60 Mon Sep 17 00:00:00 2001 From: discord9 Date: Tue, 10 Dec 2024 20:47:06 +0800 Subject: [PATCH 12/16] fix: wait for election --- Cargo.lock | 38 ++++++++++++++++++++++ src/common/meta/src/kv_backend/etcd.rs | 2 ++ tests/runner/Cargo.toml | 1 + tests/runner/src/env.rs | 9 ++++-- tests/runner/src/util.rs | 44 ++++++++++++++++++++++---- 5 files changed, 85 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ee842146f52..2c21fe1bc07e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6132,6 +6132,18 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "local-ip-address" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3669cf5561f8d27e8fc84cc15e58350e70f557d4d65f70e3154e54cd2f8e1782" +dependencies = [ + "libc", + "neli", + "thiserror 1.0.64", + "windows-sys 0.59.0", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -7108,6 +7120,31 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "neli" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1100229e06604150b3becd61a4965d5c70f3be1759544ea7274166f4be41ef43" +dependencies = [ + "byteorder", + "libc", + "log", + "neli-proc-macros", +] + +[[package]] +name = "neli-proc-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168194d373b1e134786274020dae7fc5513d565ea2ebb9bc9ff17ffb69106d4" +dependencies = [ + "either", + "proc-macro2", + "quote", + "serde", + "syn 1.0.109", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -11402,6 +11439,7 @@ dependencies = [ "datatypes", "flate2", "hex", + "local-ip-address", "mysql", "reqwest", "serde", diff --git a/src/common/meta/src/kv_backend/etcd.rs b/src/common/meta/src/kv_backend/etcd.rs index 1cdd45bc5c13..a787940b6df0 100644 --- a/src/common/meta/src/kv_backend/etcd.rs +++ b/src/common/meta/src/kv_backend/etcd.rs @@ -15,6 +15,7 @@ use std::any::Any; use std::sync::Arc; +use common_telemetry::info; use etcd_client::{ Client, DeleteOptions, GetOptions, PutOptions, Txn, TxnOp, TxnOpResponse, TxnResponse, }; @@ -55,6 +56,7 @@ impl EtcdStore { } pub fn with_etcd_client(client: Client, max_txn_ops: usize) -> KvBackendRef { + info!("Connected to etcd"); Arc::new(Self { client, max_txn_ops, diff --git a/tests/runner/Cargo.toml b/tests/runner/Cargo.toml index 148a4eba70f5..72600b0b2baa 100644 --- a/tests/runner/Cargo.toml +++ b/tests/runner/Cargo.toml @@ -23,6 +23,7 @@ tokio-postgres = { workspace = true } # sqlness 0.6.0 have a bug causing `cargo sqlness` to fail(see https://github.com/CeresDB/sqlness/issues/68) which is fixed in 0.6.1 flate2 = "1.0" hex = "0.4.3" +local-ip-address = "0.6.3" reqwest = { version = "0.12.9", default-features = false, features = ["rustls-tls"] } sha2 = "=0.10.8" sqlness = "0.6.1" diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index 9212288cd7e7..86afea0ca63b 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -368,8 +368,8 @@ impl Env { "127.0.0.1:29302".to_string(), "--server-addr".to_string(), "127.0.0.1:29302".to_string(), - "--backend".to_string(), - "memory-store".to_string(), + // "--backend".to_string(), + //"memory-store".to_string(), "--enable-region-failover".to_string(), "false".to_string(), "--http-addr=127.0.0.1:29502".to_string(), @@ -510,6 +510,9 @@ impl Env { .lock() .expect("lock poisoned") .replace(metasrv); + + // wait for metasrv to start + tokio::time::sleep(Duration::from_secs(5)).await; } let mut processes = vec![]; @@ -792,6 +795,8 @@ impl Database for GreptimeDB { } self.env.restart_server(self, true).await; + // sleep for a while to wait for the server to fully boot up + tokio::time::sleep(Duration::from_secs(5)).await; } if let Some(protocol) = ctx.context.get(PROTOCOL_KEY) { diff --git a/tests/runner/src/util.rs b/tests/runner/src/util.rs index 61bed6e233e0..9dd0bf4eb1c9 100644 --- a/tests/runner/src/util.rs +++ b/tests/runner/src/util.rs @@ -202,11 +202,10 @@ pub fn setup_etcd(client_ports: Vec, peer_port: Option, etcd_version: let etcd_version = etcd_version.unwrap_or("v3.5.17"); let etcd_image = format!("quay.io/coreos/etcd:{etcd_version}"); let peer_url = format!("http://0.0.0.0:{peer_port}"); + let my_local_ip = local_ip_address::local_ip().unwrap(); + + let my_local_ip_str = my_local_ip.to_string(); - let client_ports_fmt = client_ports - .iter() - .map(|p| format!("http://0.0.0.0:{p}")) - .collect::>(); let mut arg_list = vec![]; arg_list.extend([ "run", @@ -222,11 +221,42 @@ pub fn setup_etcd(client_ports: Vec, peer_port: Option, etcd_version: "etcd", "-name", "etcd0", - "-listen-client-urls", + "-advertise-client-urls", ]); - arg_list.extend(client_ports_fmt.iter().map(std::ops::Deref::deref)); - arg_list.extend(["-listen-peer-urls", &peer_url, "-initial-cluster-state new"]); + let adv_client_urls = client_ports + .iter() + .map(|p| format!("http://{my_local_ip_str}:{p}")) + .collect::>() + .join(","); + + arg_list.push(&adv_client_urls); + + arg_list.extend(["-listen-client-urls"]); + + let client_ports_fmt = client_ports + .iter() + .map(|p| format!("http://0.0.0.0:{p}")) + .collect::>() + .join(","); + + arg_list.push(&client_ports_fmt); + + arg_list.push("-initial-advertise-peer-urls"); + let advertise_peer_url = format!("http://{my_local_ip_str}:{peer_port}"); + arg_list.push(&advertise_peer_url); + + arg_list.extend(["-listen-peer-urls", &peer_url]); + + arg_list.extend(["-initial-cluster-token", "etcd-cluster-1"]); + + arg_list.push("-initial-cluster"); + + let init_cluster_url = format!("etcd0=http://{my_local_ip_str}:{peer_port}"); + + arg_list.push(&init_cluster_url); + + arg_list.extend(["-initial-cluster-state", "new"]); let mut cmd = std::process::Command::new("docker"); From 6258c4a3f639ca6462209752888c7159bf132e86 Mon Sep 17 00:00:00 2001 From: discord9 Date: Wed, 11 Dec 2024 11:30:10 +0800 Subject: [PATCH 13/16] fix: normal sqlness --- tests/runner/src/env.rs | 21 +++++++++++++++------ tests/runner/src/main.rs | 6 ++++-- tests/runner/src/util.rs | 4 ++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index 86afea0ca63b..6ed1152ab618 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -135,7 +135,7 @@ impl Env { self.build_db(); self.setup_wal(); - let db_ctx = GreptimeDBContext::new(self.wal.clone()); + let db_ctx = GreptimeDBContext::new(self.wal.clone(), self.store_config.clone()); let server_process = self.start_server("standalone", &db_ctx, true).await; @@ -156,7 +156,7 @@ impl Env { self.setup_wal(); self.setup_etcd(); - let db_ctx = GreptimeDBContext::new(self.wal.clone()); + let db_ctx = GreptimeDBContext::new(self.wal.clone(), self.store_config.clone()); // start a distributed GreptimeDB let meta_server = self.start_server("metasrv", &db_ctx, true).await; @@ -263,6 +263,7 @@ impl Env { time: 0, datanode_id: Default::default(), wal: self.wal.clone(), + store_config: self.store_config.clone(), }, is_standalone: false, env: self.clone(), @@ -360,7 +361,7 @@ impl Env { ) } "metasrv" => { - let args = vec![ + let mut args = vec![ DEFAULT_LOG_LEVEL.to_string(), subcommand.to_string(), "start".to_string(), @@ -368,8 +369,6 @@ impl Env { "127.0.0.1:29302".to_string(), "--server-addr".to_string(), "127.0.0.1:29302".to_string(), - // "--backend".to_string(), - //"memory-store".to_string(), "--enable-region-failover".to_string(), "false".to_string(), "--http-addr=127.0.0.1:29502".to_string(), @@ -380,6 +379,9 @@ impl Env { "-c".to_string(), self.generate_config_file(subcommand, db_ctx), ]; + if !db_ctx.store_config().setup_etcd { + args.extend(vec!["--backend".to_string(), "memory-store".to_string()]) + } (args, vec![METASRV_ADDR.to_string()]) } _ => panic!("Unexpected subcommand: {subcommand}"), @@ -512,6 +514,7 @@ impl Env { .replace(metasrv); // wait for metasrv to start + // since it seems older version of db might take longer to complete election tokio::time::sleep(Duration::from_secs(5)).await; } @@ -871,14 +874,16 @@ struct GreptimeDBContext { time: i64, datanode_id: AtomicU32, wal: WalConfig, + store_config: StoreConfig, } impl GreptimeDBContext { - pub fn new(wal: WalConfig) -> Self { + pub fn new(wal: WalConfig, store_config: StoreConfig) -> Self { Self { time: common_time::util::current_time_millis(), datanode_id: AtomicU32::new(0), wal, + store_config, } } @@ -906,6 +911,10 @@ impl GreptimeDBContext { fn reset_datanode_id(&self) { self.datanode_id.store(0, Ordering::Relaxed); } + + fn store_config(&self) -> StoreConfig { + self.store_config.clone() + } } struct ResultDisplayer { diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index aed0de953531..8ce2f7213030 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -171,8 +171,10 @@ async fn main() { // clean up and exit if !args.preserve_state { - println!("Stopping etcd"); - util::stop_etcd(); + if args.setup_etcd { + println!("Stopping etcd"); + util::stop_rm_etcd(); + } println!("Removing state in {:?}", sqlness_home); tokio::fs::remove_dir_all(sqlness_home).await.unwrap(); } diff --git a/tests/runner/src/util.rs b/tests/runner/src/util.rs index 9dd0bf4eb1c9..1376bfbcdcab 100644 --- a/tests/runner/src/util.rs +++ b/tests/runner/src/util.rs @@ -279,8 +279,8 @@ pub fn setup_etcd(client_ports: Vec, peer_port: Option, etcd_version: } } -/// Stop the etcd container -pub fn stop_etcd() { +/// Stop and remove the etcd container +pub fn stop_rm_etcd() { let status = std::process::Command::new("docker") .args(["container", "stop", "etcd"]) .status(); From e57b22d09bd85084765015a8305fc0add2fad1fc Mon Sep 17 00:00:00 2001 From: discord9 Date: Wed, 11 Dec 2024 20:22:29 +0800 Subject: [PATCH 14/16] refactor: hashmap for bin dir --- tests/runner/src/env.rs | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index 6ed1152ab618..0ae632034cda 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::borrow::Cow; +use std::collections::HashMap; use std::fmt::Display; use std::fs::OpenOptions; use std::io; @@ -81,10 +82,10 @@ pub struct Env { /// When running in CI, this is expected to be set. /// If not set, this runner will build the GreptimeDB binary itself when needed, and set this field by then. bins_dir: Arc>>, + /// The path to the directory that contains the old pre-built GreptimeDB binaries. + versioned_bins_dirs: Arc>>, /// Pull different versions of GreptimeDB on need. pull_version_on_need: bool, - /// old bins dir, useful when switching versions - old_bins_dir: Arc>>, /// Store address for metasrv metadata store_config: StoreConfig, } @@ -122,8 +123,11 @@ impl Env { server_addrs, wal, pull_version_on_need, - bins_dir: Arc::new(Mutex::new(bins_dir)), - old_bins_dir: Arc::new(Mutex::new(None)), + bins_dir: Arc::new(Mutex::new(bins_dir.clone())), + versioned_bins_dirs: Arc::new(Mutex::new(HashMap::from_iter([( + "latest".to_string(), + bins_dir.clone().unwrap_or(util::get_binary_dir("debug")), + )]))), store_config, } } @@ -782,13 +786,15 @@ impl Database for GreptimeDB { if ctx.context.contains_key("restart") && self.env.server_addrs.server_addr.is_none() { self.env.restart_server(self, false).await; } else if let Some(version) = ctx.context.get("version") { - if self.env.old_bins_dir.lock().unwrap().is_none() { - // save old bins dir - *self.env.old_bins_dir.lock().unwrap() = self.env.bins_dir.lock().unwrap().clone(); - } - if version == "latest" { - // use default latest by building db now - *self.env.bins_dir.lock().unwrap() = self.env.old_bins_dir.lock().unwrap().clone(); + let version_bin_dir = self + .env + .versioned_bins_dirs + .lock() + .expect("lock poison") + .get(version.as_str()) + .cloned(); + if let Some(path) = version_bin_dir { + *self.env.bins_dir.lock().unwrap() = Some(path.clone()); } else { // use version in dir files maybe_pull_binary(version, self.env.pull_version_on_need).await; From 5faa00736ac6069f77d1fc3cbd6b9feb01e80ccf Mon Sep 17 00:00:00 2001 From: discord9 Date: Thu, 12 Dec 2024 19:38:15 +0800 Subject: [PATCH 15/16] test: past 3 major version compat crate table --- tests/runner/src/env.rs | 32 ++-- tests/runner/src/util.rs | 9 +- .../common/table_engine_0_10_2.result | 137 ++++++++++++++++++ .../standalone/common/table_engine_0_10_2.sql | 60 ++++++++ .../common/table_engine_v0_11_0.result | 137 ++++++++++++++++++ .../common/table_engine_v0_11_0.sql | 60 ++++++++ .../common/table_engine_v0_9_5.result | 137 ++++++++++++++++++ .../standalone/common/table_engine_v0_9_5.sql | 60 ++++++++ 8 files changed, 615 insertions(+), 17 deletions(-) create mode 100644 tests/upgrade-compat/standalone/common/table_engine_0_10_2.result create mode 100644 tests/upgrade-compat/standalone/common/table_engine_0_10_2.sql create mode 100644 tests/upgrade-compat/standalone/common/table_engine_v0_11_0.result create mode 100644 tests/upgrade-compat/standalone/common/table_engine_v0_11_0.sql create mode 100644 tests/upgrade-compat/standalone/common/table_engine_v0_9_5.result create mode 100644 tests/upgrade-compat/standalone/common/table_engine_v0_9_5.sql diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index 0ae632034cda..5bda8e82e491 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -46,7 +46,7 @@ use tokio::sync::Mutex as TokioMutex; use tokio_postgres::{Client as PgClient, SimpleQueryMessage as PgRow}; use crate::protocol_interceptor::{MYSQL, PROTOCOL_KEY}; -use crate::util::{get_workspace_root, maybe_pull_binary}; +use crate::util::{get_workspace_root, maybe_pull_binary, PROGRAM}; use crate::{util, ServerAddr}; const METASRV_ADDR: &str = "127.0.0.1:29302"; @@ -400,23 +400,20 @@ impl Env { } } - #[cfg(not(windows))] - let program = "./greptime"; - #[cfg(windows)] - let program = "greptime.exe"; + let program = PROGRAM; let bins_dir = self.bins_dir.lock().unwrap().clone().expect( "GreptimeDB binary is not available. Please pass in the path to the directory that contains the pre-built GreptimeDB binary. Or you may call `self.build_db()` beforehand.", ); let mut process = Command::new(program) - .current_dir(bins_dir) + .current_dir(bins_dir.clone()) .env("TZ", "UTC") .args(args) .stdout(stdout_file) .spawn() .unwrap_or_else(|error| { - panic!("Failed to start the DB with subcommand {subcommand},Error: {error}") + panic!("Failed to start the DB with subcommand {subcommand},Error: {error}, path: {:?}", bins_dir.join(program)); }); for check_ip_addr in &check_ip_addrs { @@ -793,14 +790,19 @@ impl Database for GreptimeDB { .expect("lock poison") .get(version.as_str()) .cloned(); - if let Some(path) = version_bin_dir { - *self.env.bins_dir.lock().unwrap() = Some(path.clone()); - } else { - // use version in dir files - maybe_pull_binary(version, self.env.pull_version_on_need).await; - let root = get_workspace_root(); - let new_path = PathBuf::from_iter([&root, version]); - *self.env.bins_dir.lock().unwrap() = Some(new_path); + + match version_bin_dir { + Some(path) if path.clone().join(PROGRAM).is_file() => { + // use version in versioned_bins_dirs + *self.env.bins_dir.lock().unwrap() = Some(path.clone()); + } + _ => { + // use version in dir files + maybe_pull_binary(version, self.env.pull_version_on_need).await; + let root = get_workspace_root(); + let new_path = PathBuf::from_iter([&root, version]); + *self.env.bins_dir.lock().unwrap() = Some(new_path); + } } self.env.restart_server(self, true).await; diff --git a/tests/runner/src/util.rs b/tests/runner/src/util.rs index 1376bfbcdcab..4bcd482a26bf 100644 --- a/tests/runner/src/util.rs +++ b/tests/runner/src/util.rs @@ -27,6 +27,11 @@ use tokio_stream::StreamExt; /// Check port every 0.1 second. const PORT_CHECK_INTERVAL: Duration = Duration::from_millis(100); +#[cfg(not(windows))] +pub const PROGRAM: &str = "./greptime"; +#[cfg(windows)] +pub const PROGRAM: &str = "greptime.exe"; + fn http_proxy() -> Option { for proxy in ["http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY"] { if let Ok(proxy_addr) = std::env::var(proxy) { @@ -124,7 +129,7 @@ pub async fn pull_binary(version: &str) { println!("Downloading {version} binary from {}", url); // mkdir {version} - std::fs::create_dir(version).unwrap(); + let _ = std::fs::create_dir(version); let archive = Path::new(version).join(filename); let folder_path = Path::new(version); @@ -176,7 +181,7 @@ pub async fn pull_binary(version: &str) { /// Pull the binary if it does not exist and `pull_version_on_need` is true. pub async fn maybe_pull_binary(version: &str, pull_version_on_need: bool) { - let exist = Path::new(version).is_dir(); + let exist = Path::new(version).join(PROGRAM).is_file(); match (exist, pull_version_on_need){ (true, _) => println!("Binary {version} exists"), (false, false) => panic!("Binary {version} does not exist, please run with --pull-version-on-need or manually download it"), diff --git a/tests/upgrade-compat/standalone/common/table_engine_0_10_2.result b/tests/upgrade-compat/standalone/common/table_engine_0_10_2.result new file mode 100644 index 000000000000..046255a641f3 --- /dev/null +++ b/tests/upgrade-compat/standalone/common/table_engine_0_10_2.result @@ -0,0 +1,137 @@ +-- SQLNESS ARG version=v0.10.2 +CREATE TABLE mito_system_metrics ( + host STRING, + idc STRING, + cpu_util DOUBLE, + memory_util DOUBLE, + disk_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host, idc), + TIME INDEX(ts) +)ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO mito_system_metrics +VALUES + ("host1", "idc_a", 11.8, 10.3, 10.3, 1667446797450), + ("host2", "idc_a", 80.0, 70.3, 90.0, 1667446797450), + ("host1", "idc_b", 50.0, 66.7, 40.6, 1667446797450); + +Affected Rows: 3 + +CREATE TABLE phy (ts timestamp time index, cpu_util double) engine=metric with ("physical_metric_table" = ""); + +Affected Rows: 0 + +CREATE TABLE system_metrics ( + host STRING, + cpu_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host), + TIME INDEX(ts) +)ENGINE=metric with ("on_physical_table" = "phy"); + +Affected Rows: 0 + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host1', 11.8, 1667446797450), + ('host2', 80.0, 1667446797450), + ('host1', 50.0, 1667446797450); + +Affected Rows: 3 + +-- SQLNESS ARG version=latest +SHOW CREATE TABLE mito_system_metrics; + ++---------------------+-----------------------------------------------------------+ +| Table | Create Table | ++---------------------+-----------------------------------------------------------+ +| mito_system_metrics | CREATE TABLE IF NOT EXISTS "mito_system_metrics" ( | +| | "host" STRING NULL, | +| | "idc" STRING NULL, | +| | "cpu_util" DOUBLE NULL, | +| | "memory_util" DOUBLE NULL, | +| | "disk_util" DOUBLE NULL, | +| | "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host", "idc") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++---------------------+-----------------------------------------------------------+ + +SHOW CREATE TABLE system_metrics; + ++----------------+-----------------------------------------------------------+ +| Table | Create Table | ++----------------+-----------------------------------------------------------+ +| system_metrics | CREATE TABLE IF NOT EXISTS "system_metrics" ( | +| | "cpu_util" DOUBLE NULL, | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=metric | +| | WITH( | +| | on_physical_table = 'phy' | +| | ) | ++----------------+-----------------------------------------------------------+ + +INSERT INTO mito_system_metrics +VALUES + ("host3", "idc_a", 90.0, 70.3, 90.0, 1667446797450), + ("host4", "idc_a", 70.0, 70.3, 90.0, 1667446797450), + ("host5", "idc_a", 60.0, 70.3, 90.0, 1667446797450); + +Affected Rows: 3 + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host3', 90.0, 1667446797450), + ('host4', 70.0, 1667446797450), + ('host5', 60.0, 1667446797450); + +Affected Rows: 3 + +SELECT * FROM mito_system_metrics; + ++-------+-------+----------+-------------+-----------+-------------------------+ +| host | idc | cpu_util | memory_util | disk_util | ts | ++-------+-------+----------+-------------+-----------+-------------------------+ +| host1 | idc_a | 11.8 | 10.3 | 10.3 | 2022-11-03T03:39:57.450 | +| host1 | idc_b | 50.0 | 66.7 | 40.6 | 2022-11-03T03:39:57.450 | +| host2 | idc_a | 80.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +| host3 | idc_a | 90.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +| host4 | idc_a | 70.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +| host5 | idc_a | 60.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | ++-------+-------+----------+-------------+-----------+-------------------------+ + +SELECT * FROM system_metrics; + ++----------+-------+-------------------------+ +| cpu_util | host | ts | ++----------+-------+-------------------------+ +| 80.0 | host2 | 2022-11-03T03:39:57.450 | +| 70.0 | host4 | 2022-11-03T03:39:57.450 | +| 60.0 | host5 | 2022-11-03T03:39:57.450 | +| 90.0 | host3 | 2022-11-03T03:39:57.450 | +| 50.0 | host1 | 2022-11-03T03:39:57.450 | ++----------+-------+-------------------------+ + +DROP TABLE mito_system_metrics; + +Affected Rows: 0 + +DROP TABLE system_metrics; + +Affected Rows: 0 + +DROP TABLE phy; + +Affected Rows: 0 + diff --git a/tests/upgrade-compat/standalone/common/table_engine_0_10_2.sql b/tests/upgrade-compat/standalone/common/table_engine_0_10_2.sql new file mode 100644 index 000000000000..1907533b1592 --- /dev/null +++ b/tests/upgrade-compat/standalone/common/table_engine_0_10_2.sql @@ -0,0 +1,60 @@ +-- SQLNESS ARG version=v0.10.2 +CREATE TABLE mito_system_metrics ( + host STRING, + idc STRING, + cpu_util DOUBLE, + memory_util DOUBLE, + disk_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host, idc), + TIME INDEX(ts) +)ENGINE=mito; + +INSERT INTO mito_system_metrics +VALUES + ("host1", "idc_a", 11.8, 10.3, 10.3, 1667446797450), + ("host2", "idc_a", 80.0, 70.3, 90.0, 1667446797450), + ("host1", "idc_b", 50.0, 66.7, 40.6, 1667446797450); + +CREATE TABLE phy (ts timestamp time index, cpu_util double) engine=metric with ("physical_metric_table" = ""); + +CREATE TABLE system_metrics ( + host STRING, + cpu_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host), + TIME INDEX(ts) +)ENGINE=metric with ("on_physical_table" = "phy"); + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host1', 11.8, 1667446797450), + ('host2', 80.0, 1667446797450), + ('host1', 50.0, 1667446797450); + +-- SQLNESS ARG version=latest +SHOW CREATE TABLE mito_system_metrics; + +SHOW CREATE TABLE system_metrics; + +INSERT INTO mito_system_metrics +VALUES + ("host3", "idc_a", 90.0, 70.3, 90.0, 1667446797450), + ("host4", "idc_a", 70.0, 70.3, 90.0, 1667446797450), + ("host5", "idc_a", 60.0, 70.3, 90.0, 1667446797450); + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host3', 90.0, 1667446797450), + ('host4', 70.0, 1667446797450), + ('host5', 60.0, 1667446797450); + +SELECT * FROM mito_system_metrics; + +SELECT * FROM system_metrics; + +DROP TABLE mito_system_metrics; + +DROP TABLE system_metrics; + +DROP TABLE phy; diff --git a/tests/upgrade-compat/standalone/common/table_engine_v0_11_0.result b/tests/upgrade-compat/standalone/common/table_engine_v0_11_0.result new file mode 100644 index 000000000000..7ce230a6881d --- /dev/null +++ b/tests/upgrade-compat/standalone/common/table_engine_v0_11_0.result @@ -0,0 +1,137 @@ +-- SQLNESS ARG version=v0.11.0 +CREATE TABLE mito_system_metrics ( + host STRING, + idc STRING, + cpu_util DOUBLE, + memory_util DOUBLE, + disk_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host, idc), + TIME INDEX(ts) +)ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO mito_system_metrics +VALUES + ("host1", "idc_a", 11.8, 10.3, 10.3, 1667446797450), + ("host2", "idc_a", 80.0, 70.3, 90.0, 1667446797450), + ("host1", "idc_b", 50.0, 66.7, 40.6, 1667446797450); + +Affected Rows: 3 + +CREATE TABLE phy (ts timestamp time index, cpu_util double) engine=metric with ("physical_metric_table" = ""); + +Affected Rows: 0 + +CREATE TABLE system_metrics ( + host STRING, + cpu_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host), + TIME INDEX(ts) +)ENGINE=metric with ("on_physical_table" = "phy"); + +Affected Rows: 0 + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host1', 11.8, 1667446797450), + ('host2', 80.0, 1667446797450), + ('host1', 50.0, 1667446797450); + +Affected Rows: 3 + +-- SQLNESS ARG version=latest +SHOW CREATE TABLE mito_system_metrics; + ++---------------------+-----------------------------------------------------------+ +| Table | Create Table | ++---------------------+-----------------------------------------------------------+ +| mito_system_metrics | CREATE TABLE IF NOT EXISTS "mito_system_metrics" ( | +| | "host" STRING NULL, | +| | "idc" STRING NULL, | +| | "cpu_util" DOUBLE NULL, | +| | "memory_util" DOUBLE NULL, | +| | "disk_util" DOUBLE NULL, | +| | "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host", "idc") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++---------------------+-----------------------------------------------------------+ + +SHOW CREATE TABLE system_metrics; + ++----------------+-----------------------------------------------------------+ +| Table | Create Table | ++----------------+-----------------------------------------------------------+ +| system_metrics | CREATE TABLE IF NOT EXISTS "system_metrics" ( | +| | "cpu_util" DOUBLE NULL, | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=metric | +| | WITH( | +| | on_physical_table = 'phy' | +| | ) | ++----------------+-----------------------------------------------------------+ + +INSERT INTO mito_system_metrics +VALUES + ("host3", "idc_a", 90.0, 70.3, 90.0, 1667446797450), + ("host4", "idc_a", 70.0, 70.3, 90.0, 1667446797450), + ("host5", "idc_a", 60.0, 70.3, 90.0, 1667446797450); + +Affected Rows: 3 + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host3', 90.0, 1667446797450), + ('host4', 70.0, 1667446797450), + ('host5', 60.0, 1667446797450); + +Affected Rows: 3 + +SELECT * FROM mito_system_metrics; + ++-------+-------+----------+-------------+-----------+-------------------------+ +| host | idc | cpu_util | memory_util | disk_util | ts | ++-------+-------+----------+-------------+-----------+-------------------------+ +| host1 | idc_a | 11.8 | 10.3 | 10.3 | 2022-11-03T03:39:57.450 | +| host1 | idc_b | 50.0 | 66.7 | 40.6 | 2022-11-03T03:39:57.450 | +| host2 | idc_a | 80.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +| host3 | idc_a | 90.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +| host4 | idc_a | 70.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +| host5 | idc_a | 60.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | ++-------+-------+----------+-------------+-----------+-------------------------+ + +SELECT * FROM system_metrics; + ++----------+-------+-------------------------+ +| cpu_util | host | ts | ++----------+-------+-------------------------+ +| 80.0 | host2 | 2022-11-03T03:39:57.450 | +| 70.0 | host4 | 2022-11-03T03:39:57.450 | +| 60.0 | host5 | 2022-11-03T03:39:57.450 | +| 90.0 | host3 | 2022-11-03T03:39:57.450 | +| 50.0 | host1 | 2022-11-03T03:39:57.450 | ++----------+-------+-------------------------+ + +DROP TABLE mito_system_metrics; + +Affected Rows: 0 + +DROP TABLE system_metrics; + +Affected Rows: 0 + +DROP TABLE phy; + +Affected Rows: 0 + diff --git a/tests/upgrade-compat/standalone/common/table_engine_v0_11_0.sql b/tests/upgrade-compat/standalone/common/table_engine_v0_11_0.sql new file mode 100644 index 000000000000..963170fdf583 --- /dev/null +++ b/tests/upgrade-compat/standalone/common/table_engine_v0_11_0.sql @@ -0,0 +1,60 @@ +-- SQLNESS ARG version=v0.11.0 +CREATE TABLE mito_system_metrics ( + host STRING, + idc STRING, + cpu_util DOUBLE, + memory_util DOUBLE, + disk_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host, idc), + TIME INDEX(ts) +)ENGINE=mito; + +INSERT INTO mito_system_metrics +VALUES + ("host1", "idc_a", 11.8, 10.3, 10.3, 1667446797450), + ("host2", "idc_a", 80.0, 70.3, 90.0, 1667446797450), + ("host1", "idc_b", 50.0, 66.7, 40.6, 1667446797450); + +CREATE TABLE phy (ts timestamp time index, cpu_util double) engine=metric with ("physical_metric_table" = ""); + +CREATE TABLE system_metrics ( + host STRING, + cpu_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host), + TIME INDEX(ts) +)ENGINE=metric with ("on_physical_table" = "phy"); + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host1', 11.8, 1667446797450), + ('host2', 80.0, 1667446797450), + ('host1', 50.0, 1667446797450); + +-- SQLNESS ARG version=latest +SHOW CREATE TABLE mito_system_metrics; + +SHOW CREATE TABLE system_metrics; + +INSERT INTO mito_system_metrics +VALUES + ("host3", "idc_a", 90.0, 70.3, 90.0, 1667446797450), + ("host4", "idc_a", 70.0, 70.3, 90.0, 1667446797450), + ("host5", "idc_a", 60.0, 70.3, 90.0, 1667446797450); + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host3', 90.0, 1667446797450), + ('host4', 70.0, 1667446797450), + ('host5', 60.0, 1667446797450); + +SELECT * FROM mito_system_metrics; + +SELECT * FROM system_metrics; + +DROP TABLE mito_system_metrics; + +DROP TABLE system_metrics; + +DROP TABLE phy; diff --git a/tests/upgrade-compat/standalone/common/table_engine_v0_9_5.result b/tests/upgrade-compat/standalone/common/table_engine_v0_9_5.result new file mode 100644 index 000000000000..41b81f01c082 --- /dev/null +++ b/tests/upgrade-compat/standalone/common/table_engine_v0_9_5.result @@ -0,0 +1,137 @@ +-- SQLNESS ARG version=v0.9.5 +CREATE TABLE mito_system_metrics ( + host STRING, + idc STRING, + cpu_util DOUBLE, + memory_util DOUBLE, + disk_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host, idc), + TIME INDEX(ts) +)ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO mito_system_metrics +VALUES + ("host1", "idc_a", 11.8, 10.3, 10.3, 1667446797450), + ("host2", "idc_a", 80.0, 70.3, 90.0, 1667446797450), + ("host1", "idc_b", 50.0, 66.7, 40.6, 1667446797450); + +Affected Rows: 3 + +CREATE TABLE phy (ts timestamp time index, cpu_util double) engine=metric with ("physical_metric_table" = ""); + +Affected Rows: 0 + +CREATE TABLE system_metrics ( + host STRING, + cpu_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host), + TIME INDEX(ts) +)ENGINE=metric with ("on_physical_table" = "phy"); + +Affected Rows: 0 + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host1', 11.8, 1667446797450), + ('host2', 80.0, 1667446797450), + ('host1', 50.0, 1667446797450); + +Affected Rows: 3 + +-- SQLNESS ARG version=latest +SHOW CREATE TABLE mito_system_metrics; + ++---------------------+-----------------------------------------------------------+ +| Table | Create Table | ++---------------------+-----------------------------------------------------------+ +| mito_system_metrics | CREATE TABLE IF NOT EXISTS "mito_system_metrics" ( | +| | "host" STRING NULL, | +| | "idc" STRING NULL, | +| | "cpu_util" DOUBLE NULL, | +| | "memory_util" DOUBLE NULL, | +| | "disk_util" DOUBLE NULL, | +| | "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host", "idc") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++---------------------+-----------------------------------------------------------+ + +SHOW CREATE TABLE system_metrics; + ++----------------+-----------------------------------------------------------+ +| Table | Create Table | ++----------------+-----------------------------------------------------------+ +| system_metrics | CREATE TABLE IF NOT EXISTS "system_metrics" ( | +| | "cpu_util" DOUBLE NULL, | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=metric | +| | WITH( | +| | on_physical_table = 'phy' | +| | ) | ++----------------+-----------------------------------------------------------+ + +INSERT INTO mito_system_metrics +VALUES + ("host3", "idc_a", 90.0, 70.3, 90.0, 1667446797450), + ("host4", "idc_a", 70.0, 70.3, 90.0, 1667446797450), + ("host5", "idc_a", 60.0, 70.3, 90.0, 1667446797450); + +Affected Rows: 3 + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host3', 90.0, 1667446797450), + ('host4', 70.0, 1667446797450), + ('host5', 60.0, 1667446797450); + +Affected Rows: 3 + +SELECT * FROM mito_system_metrics; + ++-------+-------+----------+-------------+-----------+-------------------------+ +| host | idc | cpu_util | memory_util | disk_util | ts | ++-------+-------+----------+-------------+-----------+-------------------------+ +| host1 | idc_a | 11.8 | 10.3 | 10.3 | 2022-11-03T03:39:57.450 | +| host1 | idc_b | 50.0 | 66.7 | 40.6 | 2022-11-03T03:39:57.450 | +| host2 | idc_a | 80.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +| host3 | idc_a | 90.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +| host4 | idc_a | 70.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +| host5 | idc_a | 60.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | ++-------+-------+----------+-------------+-----------+-------------------------+ + +SELECT * FROM system_metrics; + ++----------+-------+-------------------------+ +| cpu_util | host | ts | ++----------+-------+-------------------------+ +| 80.0 | host2 | 2022-11-03T03:39:57.450 | +| 70.0 | host4 | 2022-11-03T03:39:57.450 | +| 60.0 | host5 | 2022-11-03T03:39:57.450 | +| 90.0 | host3 | 2022-11-03T03:39:57.450 | +| 50.0 | host1 | 2022-11-03T03:39:57.450 | ++----------+-------+-------------------------+ + +DROP TABLE mito_system_metrics; + +Affected Rows: 0 + +DROP TABLE system_metrics; + +Affected Rows: 0 + +DROP TABLE phy; + +Affected Rows: 0 + diff --git a/tests/upgrade-compat/standalone/common/table_engine_v0_9_5.sql b/tests/upgrade-compat/standalone/common/table_engine_v0_9_5.sql new file mode 100644 index 000000000000..9908085213ed --- /dev/null +++ b/tests/upgrade-compat/standalone/common/table_engine_v0_9_5.sql @@ -0,0 +1,60 @@ +-- SQLNESS ARG version=v0.9.5 +CREATE TABLE mito_system_metrics ( + host STRING, + idc STRING, + cpu_util DOUBLE, + memory_util DOUBLE, + disk_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host, idc), + TIME INDEX(ts) +)ENGINE=mito; + +INSERT INTO mito_system_metrics +VALUES + ("host1", "idc_a", 11.8, 10.3, 10.3, 1667446797450), + ("host2", "idc_a", 80.0, 70.3, 90.0, 1667446797450), + ("host1", "idc_b", 50.0, 66.7, 40.6, 1667446797450); + +CREATE TABLE phy (ts timestamp time index, cpu_util double) engine=metric with ("physical_metric_table" = ""); + +CREATE TABLE system_metrics ( + host STRING, + cpu_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host), + TIME INDEX(ts) +)ENGINE=metric with ("on_physical_table" = "phy"); + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host1', 11.8, 1667446797450), + ('host2', 80.0, 1667446797450), + ('host1', 50.0, 1667446797450); + +-- SQLNESS ARG version=latest +SHOW CREATE TABLE mito_system_metrics; + +SHOW CREATE TABLE system_metrics; + +INSERT INTO mito_system_metrics +VALUES + ("host3", "idc_a", 90.0, 70.3, 90.0, 1667446797450), + ("host4", "idc_a", 70.0, 70.3, 90.0, 1667446797450), + ("host5", "idc_a", 60.0, 70.3, 90.0, 1667446797450); + +INSERT INTO system_metrics (host, cpu_util, ts) +VALUES + ('host3', 90.0, 1667446797450), + ('host4', 70.0, 1667446797450), + ('host5', 60.0, 1667446797450); + +SELECT * FROM mito_system_metrics; + +SELECT * FROM system_metrics; + +DROP TABLE mito_system_metrics; + +DROP TABLE system_metrics; + +DROP TABLE phy; From 3ebdb23a4c9bb7e1220487b882018ad6b1df5c18 Mon Sep 17 00:00:00 2001 From: discord9 Date: Fri, 13 Dec 2024 15:34:09 +0800 Subject: [PATCH 16/16] refactor: allow using without setup etcd --- tests/conf/metasrv-test.toml.template | 2 +- tests/runner/Cargo.toml | 17 ++++++++--------- tests/runner/src/env.rs | 6 +++--- tests/runner/src/main.rs | 6 +++--- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/conf/metasrv-test.toml.template b/tests/conf/metasrv-test.toml.template index 5519cc65e1a3..1196403a2648 100644 --- a/tests/conf/metasrv-test.toml.template +++ b/tests/conf/metasrv-test.toml.template @@ -1,5 +1,5 @@ flush_stats_factor = 1 -{{ if setup_etcd }} +{{ if use_etcd }} ## Store server address default to etcd store. store_addrs = [{store_addrs | unescaped}] diff --git a/tests/runner/Cargo.toml b/tests/runner/Cargo.toml index 72600b0b2baa..3ea403e862e0 100644 --- a/tests/runner/Cargo.toml +++ b/tests/runner/Cargo.toml @@ -16,19 +16,18 @@ common-query.workspace = true common-recordbatch.workspace = true common-time.workspace = true datatypes = { workspace = true } +flate2 = "1.0" +hex = "0.4" +local-ip-address = "0.6" mysql = { version = "25.0.1", default-features = false, features = ["minimal", "rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } serde.workspace = true serde_json.workspace = true -tokio-postgres = { workspace = true } -# sqlness 0.6.0 have a bug causing `cargo sqlness` to fail(see https://github.com/CeresDB/sqlness/issues/68) which is fixed in 0.6.1 -flate2 = "1.0" -hex = "0.4.3" -local-ip-address = "0.6.3" -reqwest = { version = "0.12.9", default-features = false, features = ["rustls-tls"] } -sha2 = "=0.10.8" -sqlness = "0.6.1" -tar = "0.4.43" +sha2 = "0.10" +sqlness = "0.6.1" # sqlness 0.6.0 have a bug causing `cargo sqlness` to fail(see https://github.com/CeresDB/sqlness/issues/68) which is fixed in 0.6.1 +tar = "0.4" tempfile.workspace = true tinytemplate = "1.2" tokio.workspace = true +tokio-postgres = { workspace = true } tokio-stream.workspace = true diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs index 5bda8e82e491..81bbe2fb0b07 100644 --- a/tests/runner/src/env.rs +++ b/tests/runner/src/env.rs @@ -383,7 +383,7 @@ impl Env { "-c".to_string(), self.generate_config_file(subcommand, db_ctx), ]; - if !db_ctx.store_config().setup_etcd { + if db_ctx.store_config().store_addrs.is_empty() { args.extend(vec!["--backend".to_string(), "memory-store".to_string()]) } (args, vec![METASRV_ADDR.to_string()]) @@ -586,7 +586,7 @@ impl Env { procedure_dir: String, is_raft_engine: bool, kafka_wal_broker_endpoints: String, - setup_etcd: bool, + use_etcd: bool, store_addrs: String, } @@ -601,7 +601,7 @@ impl Env { procedure_dir, is_raft_engine: db_ctx.is_raft_engine(), kafka_wal_broker_endpoints: db_ctx.kafka_wal_broker_endpoints(), - setup_etcd: self.store_config.setup_etcd, + use_etcd: !self.store_config.store_addrs.is_empty(), store_addrs: self .store_config .store_addrs diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index 8ce2f7213030..2e3158e1953b 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -99,11 +99,11 @@ struct Args { #[clap(long, default_value = "true")] pull_version_on_need: bool, - /// The store addresses for the raft engine. - #[clap(long, default_value = "0.0.0.0:2379")] + /// The store addresses for metadata, if empty, will use memory store. + #[clap(long)] store_addrs: Vec, - /// Whether to setup etcd, by default it is true. + /// Whether to setup etcd, by default it is false. #[clap(long, default_value = "false")] setup_etcd: bool, }