From e02dd9a6b3d964658b24366f8298552ea5176204 Mon Sep 17 00:00:00 2001 From: Willem Wyndham Date: Mon, 22 Jan 2024 21:18:47 -0500 Subject: [PATCH 1/5] feat: first try using testcontainers --- Cargo.lock | 218 ++++++++++++++++++- cmd/crates/soroban-test/Cargo.toml | 5 + cmd/crates/soroban-test/src/lib.rs | 1 + cmd/crates/soroban-test/src/module.rs | 47 ++++ cmd/soroban-rpc/internal/test/integration.go | 2 +- 5 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 cmd/crates/soroban-test/src/module.rs diff --git a/Cargo.lock b/Cargo.lock index 192b4d2db..fe79c6f84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -241,6 +241,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard-stubs" +version = "1.42.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed59b5c00048f48d7af971b71f800fdf23e858844a6f9e4d32ca72e9399e7864" +dependencies = [ + "serde", + "serde_with 1.14.0", +] + [[package]] name = "bstr" version = "1.8.0" @@ -434,6 +444,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "const-oid" version = "0.9.5" @@ -686,14 +706,38 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core 0.13.4", + "darling_macro 0.13.4", +] + [[package]] name = "darling" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.3", + "darling_macro 0.20.3", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", ] [[package]] @@ -710,13 +754,24 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core 0.13.4", + "quote", + "syn 1.0.109", +] + [[package]] name = "darling_macro" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ - "darling_core", + "darling_core 0.20.3", "quote", "syn 2.0.39", ] @@ -940,6 +995,19 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1075,6 +1143,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.29" @@ -1082,6 +1165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1090,12 +1174,34 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "futures-sink" version = "0.3.29" @@ -1114,8 +1220,10 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -2063,6 +2171,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.27" @@ -2257,6 +2371,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is-terminal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -2898,6 +3023,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "pretty_env_logger" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c" +dependencies = [ + "env_logger", + "log", +] + [[package]] name = "prettyplease" version = "0.2.15" @@ -2974,6 +3109,21 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redis" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c580d9cbbe1d1b479e8d67cf9daf6a62c957e6846048408b80b43ac3f6af84cd" +dependencies = [ + "combine", + "itoa", + "percent-encoding", + "ryu", + "sha1_smol", + "socket2 0.4.10", + "url", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -3375,6 +3525,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +dependencies = [ + "serde", + "serde_with_macros 1.5.2", +] + [[package]] name = "serde_with" version = "3.4.0" @@ -3388,17 +3548,29 @@ dependencies = [ "indexmap 2.1.0", "serde", "serde_json", - "serde_with_macros", + "serde_with_macros 3.4.0", "time", ] +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling 0.13.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "serde_with_macros" version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" dependencies = [ - "darling", + "darling 0.20.3", "proc-macro2", "quote", "syn 2.0.39", @@ -3685,7 +3857,7 @@ source = "git+https://github.com/stellar/rs-soroban-sdk?rev=e6c2c900ab82b5f6eec4 dependencies = [ "serde", "serde_json", - "serde_with", + "serde_with 3.4.0", "soroban-env-common", "soroban-env-host", "thiserror", @@ -3716,7 +3888,7 @@ version = "20.1.0" source = "git+https://github.com/stellar/rs-soroban-sdk?rev=e6c2c900ab82b5f6eec48f69cb2cb519e19819cb#e6c2c900ab82b5f6eec48f69cb2cb519e19819cb" dependencies = [ "crate-git-revision 0.0.6", - "darling", + "darling 0.20.3", "itertools 0.11.0", "proc-macro2", "quote", @@ -3828,6 +4000,8 @@ dependencies = [ "assert_fs", "fs_extra", "predicates 2.1.5", + "pretty_env_logger", + "redis", "sep5", "serde_json", "sha2 0.10.8", @@ -3838,6 +4012,8 @@ dependencies = [ "soroban-spec", "soroban-spec-tools", "stellar-strkey 0.0.7", + "testcontainers", + "testcontainers-modules", "thiserror", "tokio", "which", @@ -3920,7 +4096,7 @@ dependencies = [ "hex", "serde", "serde_json", - "serde_with", + "serde_with 3.4.0", "stellar-strkey 0.0.8", "thiserror", ] @@ -4085,6 +4261,32 @@ dependencies = [ "soroban-sdk", ] +[[package]] +name = "testcontainers" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d2931d7f521af5bae989f716c3fa43a6af9af7ec7a5e21b59ae40878cec00" +dependencies = [ + "bollard-stubs", + "futures", + "hex", + "hmac 0.12.1", + "log", + "rand", + "serde", + "serde_json", + "sha2 0.10.8", +] + +[[package]] +name = "testcontainers-modules" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c461fd6956a37f00d9d90bc7ba9ec5b1acdf1b9ef8620417316f1125d3c4ff3" +dependencies = [ + "testcontainers", +] + [[package]] name = "thiserror" version = "1.0.50" diff --git a/cmd/crates/soroban-test/Cargo.toml b/cmd/crates/soroban-test/Cargo.toml index 649a37e57..6cb7a3d32 100644 --- a/cmd/crates/soroban-test/Cargo.toml +++ b/cmd/crates/soroban-test/Cargo.toml @@ -32,11 +32,16 @@ assert_cmd = "2.0.4" assert_fs = "1.0.7" predicates = "2.1.5" fs_extra = "1.3.0" +testcontainers = "0.15.0" +testcontainers-modules.version = "0.3.1" +testcontainers-modules.features = ["redis"] [dev-dependencies] serde_json = "1.0.93" which = { workspace = true } tokio = "1.28.1" +redis = "0.24.0" +pretty_env_logger = "0.5.0" [features] integration = [] diff --git a/cmd/crates/soroban-test/src/lib.rs b/cmd/crates/soroban-test/src/lib.rs index bda6ec420..fc01d0620 100644 --- a/cmd/crates/soroban-test/src/lib.rs +++ b/cmd/crates/soroban-test/src/lib.rs @@ -34,6 +34,7 @@ use soroban_cli::{ CommandParser, Pwd, }; +mod module; mod wasm; pub use wasm::Wasm; diff --git a/cmd/crates/soroban-test/src/module.rs b/cmd/crates/soroban-test/src/module.rs new file mode 100644 index 000000000..02012efe7 --- /dev/null +++ b/cmd/crates/soroban-test/src/module.rs @@ -0,0 +1,47 @@ +use testcontainers::{core::WaitFor, Image}; + +const NAME: &str = "redis"; +const TAG: &str = "5.0"; + +#[derive(Debug, Default)] +pub struct Soroban; + +impl Image for Soroban { + type Args = (); + + fn name(&self) -> String { + NAME.to_owned() + } + + fn tag(&self) -> String { + TAG.to_owned() + } + + fn ready_conditions(&self) -> Vec { + vec![WaitFor::message_on_stdout("Ready to accept connections")] + } +} + +#[cfg(test)] +mod tests { + use redis::Commands; + use testcontainers::clients; + + use testcontainers_modules::redis::Redis; + + #[test] + fn redis_fetch_an_integer() { + let _ = pretty_env_logger::try_init(); + let docker = clients::Cli::default(); + let node = docker.run(Redis); + let host_port = node.get_host_port_ipv4(6379); + let url = format!("redis://127.0.0.1:{host_port}"); + + let client = redis::Client::open(url.as_ref()).unwrap(); + let mut con = client.get_connection().unwrap(); + + con.set::<_, _, ()>("my_key", 42).unwrap(); + let result: i64 = con.get("my_key").unwrap(); + assert_eq!(42, result); + } +} diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go index ea918d13d..10b24c854 100644 --- a/cmd/soroban-rpc/internal/test/integration.go +++ b/cmd/soroban-rpc/internal/test/integration.go @@ -35,7 +35,7 @@ const ( friendbotURL = "http://localhost:8000/friendbot" // Needed when Core is run with ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true checkpointFrequency = 8 - sorobanRPCPort = 8000 + sorobanRPCPort = 8042 adminPort = 8080 helloWorldContractPath = "../../../../target/wasm32-unknown-unknown/test-wasms/test_hello_world.wasm" ) From a76499744e8e486d030cda702b20b5089f9595f0 Mon Sep 17 00:00:00 2001 From: Willem Wyndham Date: Tue, 23 Jan 2024 18:28:01 -0500 Subject: [PATCH 2/5] fix: got it working about half the time --- cmd/crates/soroban-test/src/module.rs | 76 ++++++++++++++++++++------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/cmd/crates/soroban-test/src/module.rs b/cmd/crates/soroban-test/src/module.rs index 02012efe7..5fa09c0d2 100644 --- a/cmd/crates/soroban-test/src/module.rs +++ b/cmd/crates/soroban-test/src/module.rs @@ -1,10 +1,32 @@ +use std::collections::HashMap; + use testcontainers::{core::WaitFor, Image}; -const NAME: &str = "redis"; -const TAG: &str = "5.0"; +const NAME: &str = "stellar/quickstart"; +// const TAG: &str = "testing"; +const TAG: &str = + "soroban-dev@sha256:0ad51035cf7caba2fd99c7c1fad0945df6932be7d5c893e1520ccdef7d6a6ffe"; #[derive(Debug, Default)] -pub struct Soroban; +pub struct Soroban(HashMap, HashMap); + +impl Soroban { + pub fn new() -> Self { + let mut map = HashMap::new(); + map.insert("ENABLE_SOROBAN_RPC".to_string(), "true".to_string()); + map.insert( + "ENABLE_SOROBAN_DIAGNOSTIC_EVENTS".to_string(), + "true".to_string(), + ); + map.insert("ENABLE_LOGS".to_string(), "true".to_string()); + map.insert("NETWORK".to_string(), "local".to_string()); + map.insert("POSTGRES_PASSWORD".to_string(), "p".to_string()); + #[allow(unused_mut)] + let mut volumes = HashMap::new(); + // volumes.insert("/home/willem/c/s/soroban-tools/opt/stellar".to_string(), "/opt/stellar".to_string()); + Soroban(map, volumes) + } +} impl Image for Soroban { type Args = (); @@ -17,31 +39,49 @@ impl Image for Soroban { TAG.to_owned() } + fn expose_ports(&self) -> Vec { + vec![8000, 11626] + } + fn ready_conditions(&self) -> Vec { - vec![WaitFor::message_on_stdout("Ready to accept connections")] + // vec![WaitFor::seconds(30)] + + vec![WaitFor::message_on_stdout("friendbot: started")] + } + + fn env_vars(&self) -> Box + '_> { + Box::new(self.0.iter()) + } + + fn volumes(&self) -> Box + '_> { + Box::new(self.1.iter()) } } #[cfg(test)] mod tests { - use redis::Commands; + use std::thread::sleep; + use std::time::Duration; + + use soroban_cli::rpc::Client; + use super::Soroban; use testcontainers::clients; - use testcontainers_modules::redis::Redis; + // use testcontainers_modules::redis::Redis; - #[test] - fn redis_fetch_an_integer() { + #[tokio::test] + async fn testcontainers_work() { let _ = pretty_env_logger::try_init(); let docker = clients::Cli::default(); - let node = docker.run(Redis); - let host_port = node.get_host_port_ipv4(6379); - let url = format!("redis://127.0.0.1:{host_port}"); - - let client = redis::Client::open(url.as_ref()).unwrap(); - let mut con = client.get_connection().unwrap(); - - con.set::<_, _, ()>("my_key", 42).unwrap(); - let result: i64 = con.get("my_key").unwrap(); - assert_eq!(42, result); + let node = docker.run(Soroban::new()); + let host_port = node.get_host_port_ipv4(8000); + let url: String = format!("http://[::1]:{host_port}/soroban/rpc"); + println!("{url}"); + // sleep(Duration::from_secs(2000)); + let client = Client::new(&url).unwrap(); + for _ in 0..20 { + sleep(Duration::from_secs(1)); + println!("{:#?}", client.get_network().await); + } } } From d5a4c6c6bd004030ab0e5b3b6435c0d4319fd1c4 Mon Sep 17 00:00:00 2001 From: Willem Wyndham Date: Thu, 25 Jan 2024 14:44:23 -0500 Subject: [PATCH 3/5] fix: use localhost and fix fmt --- cmd/crates/soroban-test/src/lib.rs | 2 +- cmd/crates/soroban-test/src/module.rs | 17 +++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cmd/crates/soroban-test/src/lib.rs b/cmd/crates/soroban-test/src/lib.rs index fc01d0620..42fdf3e1f 100644 --- a/cmd/crates/soroban-test/src/lib.rs +++ b/cmd/crates/soroban-test/src/lib.rs @@ -34,7 +34,7 @@ use soroban_cli::{ CommandParser, Pwd, }; -mod module; +pub mod module; mod wasm; pub use wasm::Wasm; diff --git a/cmd/crates/soroban-test/src/module.rs b/cmd/crates/soroban-test/src/module.rs index 5fa09c0d2..b5a0f2dea 100644 --- a/cmd/crates/soroban-test/src/module.rs +++ b/cmd/crates/soroban-test/src/module.rs @@ -60,28 +60,21 @@ impl Image for Soroban { #[cfg(test)] mod tests { - use std::thread::sleep; - use std::time::Duration; - - use soroban_cli::rpc::Client; use super::Soroban; + use soroban_cli::rpc::Client; use testcontainers::clients; - // use testcontainers_modules::redis::Redis; - #[tokio::test] async fn testcontainers_work() { let _ = pretty_env_logger::try_init(); let docker = clients::Cli::default(); let node = docker.run(Soroban::new()); let host_port = node.get_host_port_ipv4(8000); - let url: String = format!("http://[::1]:{host_port}/soroban/rpc"); + let url: String = format!("http://localhost:{host_port}/soroban/rpc"); println!("{url}"); - // sleep(Duration::from_secs(2000)); let client = Client::new(&url).unwrap(); - for _ in 0..20 { - sleep(Duration::from_secs(1)); - println!("{:#?}", client.get_network().await); - } + let res = client.get_network().await; + println!("{res:#?}"); + assert!(res.is_ok()); } } From db8681dd8eb23e003a387585c2a4040f100caabf Mon Sep 17 00:00:00 2001 From: Willem Wyndham Date: Fri, 26 Jan 2024 12:47:43 -0500 Subject: [PATCH 4/5] fix: update github actions to allow access to local docker --- .github/workflows/rust.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9028df78e..e72f8fb09 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -74,6 +74,13 @@ jobs: run: sudo apt-get update && sudo apt-get -y install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - run: cargo clippy --all-targets --target ${{ matrix.target }} - run: make build-test + - name: Enable Docker Remote API on Localhost + shell: bash + run: | + sudo mkdir -p /etc/systemd/system/docker.service.d/ + sudo cp ./.github/docker.override.conf /etc/systemd/system/docker.service.d/override.conf + sudo systemctl daemon-reload + sudo systemctl restart docker - if: startsWith(matrix.target, 'x86_64') # specify directories explicitly to avoid building the preflight library (otherwise it will fail with missing symbols) run: | From 94a364a2d2196377fc71deaacd9a8ec17c7354bb Mon Sep 17 00:00:00 2001 From: Willem Wyndham Date: Fri, 26 Jan 2024 12:51:07 -0500 Subject: [PATCH 5/5] fix: rust.yml --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e72f8fb09..e5f19092e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -75,8 +75,8 @@ jobs: - run: cargo clippy --all-targets --target ${{ matrix.target }} - run: make build-test - name: Enable Docker Remote API on Localhost - shell: bash - run: | + shell: bash + run: | sudo mkdir -p /etc/systemd/system/docker.service.d/ sudo cp ./.github/docker.override.conf /etc/systemd/system/docker.service.d/override.conf sudo systemctl daemon-reload