rewrite some flaky zombienet polkadot tests to zombienet-sdk #6757
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| // Copyright (C) Parity Technologies (UK) Ltd. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // Test that a parachain that uses a basic collator (like adder-collator) with elastic scaling | ||
| // can achieve full throughput of 3 candidates per block. | ||
|
|
||
| use anyhow::anyhow; | ||
|
|
||
| use crate::helpers::{ | ||
| assert_para_throughput, rococo, | ||
| rococo::runtime_types::{ | ||
| pallet_broker::coretime_interface::CoreAssignment, | ||
| polkadot_runtime_parachains::assigner_coretime::PartsOf57600, | ||
| }, | ||
| }; | ||
| use serde_json::json; | ||
| use subxt::{OnlineClient, PolkadotConfig}; | ||
| use subxt_signer::sr25519::dev; | ||
| use zombienet_sdk::NetworkConfigBuilder; | ||
|
|
||
| #[tokio::test(flavor = "multi_thread")] | ||
| async fn basic_3cores_test() -> Result<(), anyhow::Error> { | ||
| let _ = env_logger::try_init_from_env( | ||
| env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"), | ||
| ); | ||
|
|
||
| let images = zombienet_sdk::environment::get_images_from_env(); | ||
|
|
||
| let config = NetworkConfigBuilder::new() | ||
| .with_relaychain(|r| { | ||
| let r = r | ||
| .with_chain("rococo-local") | ||
| .with_default_command("polkadot") | ||
| .with_default_image(images.polkadot.as_str()) | ||
| .with_default_args(vec![("-lparachain=debug").into()]) | ||
| .with_genesis_overrides(json!({ | ||
| "configuration": { | ||
| "config": { | ||
| "scheduler_params": { | ||
| "num_cores": 2, | ||
| "max_validators_per_core": 1 | ||
| }, | ||
| "async_backing_params": { | ||
| "max_candidate_depth": 6, | ||
| "allowed_ancestry_len": 2 | ||
| } | ||
| } | ||
| } | ||
| })) | ||
| // Have to set a `with_node` outside of the loop below, so that `r` has the right | ||
| // type. | ||
| .with_node(|node| node.with_name("validator-0")); | ||
|
|
||
| (1..4).fold(r, |acc, i| acc.with_node(|node| node.with_name(&format!("validator-{i}")))) | ||
| }) | ||
| .with_parachain(|p| { | ||
| p.with_id(2000) | ||
| .with_default_command("adder-collator") | ||
| .cumulus_based(false) | ||
| .with_default_image(images.cumulus.as_str()) | ||
| .with_default_args(vec![("-lparachain=debug").into()]) | ||
| .with_collator(|n| n.with_name("adder-2000")) | ||
| }) | ||
| .with_parachain(|p| { | ||
| p.with_id(2001) | ||
| .with_default_command("adder-collator") | ||
| .cumulus_based(false) | ||
| .with_default_image(images.cumulus.as_str()) | ||
| .with_default_args(vec![("-lparachain=debug").into()]) | ||
| .with_collator(|n| n.with_name("adder-2001")) | ||
| }) | ||
| .build() | ||
| .map_err(|e| { | ||
| let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" "); | ||
| anyhow!("config errs: {errs}") | ||
| })?; | ||
|
|
||
| let spawn_fn = zombienet_sdk::environment::get_spawn_fn(); | ||
| let network = spawn_fn(config).await?; | ||
|
|
||
| let relay_node = network.get_node("validator-0")?; | ||
|
|
||
| let relay_client: OnlineClient<PolkadotConfig> = relay_node.wait_client().await?; | ||
| let alice = dev::alice(); | ||
|
|
||
| // Assign two extra cores to adder-2000. | ||
| relay_client | ||
| .tx() | ||
| .sign_and_submit_then_watch_default( | ||
| &rococo::tx() | ||
| .sudo() | ||
| .sudo(rococo::runtime_types::rococo_runtime::RuntimeCall::Utility( | ||
| rococo::runtime_types::pallet_utility::pallet::Call::batch { | ||
| calls: vec![ | ||
| rococo::runtime_types::rococo_runtime::RuntimeCall::Coretime( | ||
| rococo::runtime_types::polkadot_runtime_parachains::coretime::pallet::Call::assign_core { | ||
| core: 0, | ||
| begin: 0, | ||
| assignment: vec![(CoreAssignment::Task(2000), PartsOf57600(57600))], | ||
| end_hint: None | ||
| } | ||
| ), | ||
| rococo::runtime_types::rococo_runtime::RuntimeCall::Coretime( | ||
| rococo::runtime_types::polkadot_runtime_parachains::coretime::pallet::Call::assign_core { | ||
| core: 1, | ||
| begin: 0, | ||
| assignment: vec![(CoreAssignment::Task(2000), PartsOf57600(57600))], | ||
| end_hint: None | ||
| } | ||
| ), | ||
| ], | ||
| }, | ||
| )), | ||
| &alice, | ||
| ) | ||
| .await? | ||
| .wait_for_finalized_success() | ||
| .await?; | ||
|
|
||
| log::info!("2 more cores assigned to adder-2000"); | ||
|
|
||
| assert_para_throughput( | ||
| &relay_client, | ||
| 15, | ||
| [(2000, 40..46), (2001, 12..16)].into_iter().collect(), | ||
| ) | ||
| .await?; | ||
|
|
||
| log::info!("Test finished successfully"); | ||
|
|
||
| Ok(()) | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| // Copyright (C) Parity Technologies (UK) Ltd. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // Test that a paraid that doesn't use elastic scaling which acquired multiple cores does not brick | ||
| // itself if ElasticScalingMVP feature is enabled in genesis. | ||
|
|
||
| use anyhow::anyhow; | ||
|
|
||
| use crate::helpers::{ | ||
| assert_finalized_block_height, assert_para_throughput, rococo, | ||
| rococo::runtime_types::{ | ||
| pallet_broker::coretime_interface::CoreAssignment, | ||
| polkadot_runtime_parachains::assigner_coretime::PartsOf57600, | ||
| }, | ||
| }; | ||
| use serde_json::json; | ||
| use subxt::{OnlineClient, PolkadotConfig}; | ||
| use subxt_signer::sr25519::dev; | ||
| use zombienet_sdk::NetworkConfigBuilder; | ||
|
|
||
| #[tokio::test(flavor = "multi_thread")] | ||
| async fn doesnt_break_parachains_test() -> Result<(), anyhow::Error> { | ||
| let _ = env_logger::try_init_from_env( | ||
| env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"), | ||
| ); | ||
|
|
||
| let images = zombienet_sdk::environment::get_images_from_env(); | ||
|
|
||
| let config = NetworkConfigBuilder::new() | ||
| .with_relaychain(|r| { | ||
| let r = r | ||
| .with_chain("rococo-local") | ||
| .with_default_command("polkadot") | ||
| .with_default_image(images.polkadot.as_str()) | ||
| .with_default_args(vec![("-lparachain=debug").into()]) | ||
| .with_genesis_overrides(json!({ | ||
| "configuration": { | ||
| "config": { | ||
| "scheduler_params": { | ||
| "num_cores": 1, | ||
| "max_validators_per_core": 2 | ||
| }, | ||
| "async_backing_params": { | ||
| "max_candidate_depth": 6, | ||
| "allowed_ancestry_len": 2 | ||
| } | ||
| } | ||
| } | ||
| })) | ||
| // Have to set a `with_node` outside of the loop below, so that `r` has the right | ||
| // type. | ||
| .with_node(|node| node.with_name("validator-0")); | ||
|
|
||
| (1..4).fold(r, |acc, i| acc.with_node(|node| node.with_name(&format!("validator-{i}")))) | ||
| }) | ||
| .with_parachain(|p| { | ||
| // Use rococo-parachain default, which has 6 second slot time. Also, don't use | ||
| // slot-based collator. | ||
| p.with_id(2000) | ||
| .with_default_command("polkadot-parachain") | ||
| .with_default_image(images.cumulus.as_str()) | ||
| .with_default_args(vec![("-lparachain=debug,aura=debug").into()]) | ||
| .with_collator(|n| n.with_name("collator-2000")) | ||
| }) | ||
| .build() | ||
| .map_err(|e| { | ||
| let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" "); | ||
| anyhow!("config errs: {errs}") | ||
| })?; | ||
|
|
||
| let spawn_fn = zombienet_sdk::environment::get_spawn_fn(); | ||
| let network = spawn_fn(config).await?; | ||
|
|
||
| let relay_node = network.get_node("validator-0")?; | ||
| let para_node = network.get_node("collator-2000")?; | ||
|
|
||
| let relay_client: OnlineClient<PolkadotConfig> = relay_node.wait_client().await?; | ||
| let alice = dev::alice(); | ||
|
|
||
| relay_client | ||
| .tx() | ||
| .sign_and_submit_then_watch_default( | ||
| &rococo::tx() | ||
| .sudo() | ||
| .sudo(rococo::runtime_types::rococo_runtime::RuntimeCall::Coretime( | ||
| rococo::runtime_types::polkadot_runtime_parachains::coretime::pallet::Call::assign_core { | ||
| core: 0, | ||
| begin: 0, | ||
| assignment: vec![(CoreAssignment::Task(2000), PartsOf57600(57600))], | ||
| end_hint: None | ||
| } | ||
| )), | ||
| &alice, | ||
| ) | ||
| .await? | ||
| .wait_for_finalized_success() | ||
| .await?; | ||
|
|
||
| log::info!("1 more core assigned to the parachain"); | ||
|
There was a problem hiding this comment. In the previous test we had the assurance that adding additional cores worked because of the throughput assertion. Here we don't have this. I would add a check that the para has indeed two cores (and is still working with the throughput of 1) |
||
|
|
||
| // Expect the parachain to be making normal progress, 1 candidate backed per relay chain block. | ||
| assert_para_throughput(&relay_client, 15, [(2000, 13..16)].into_iter().collect()).await?; | ||
|
|
||
| let para_client = para_node.wait_client().await?; | ||
| // Assert the parachain finalized block height is also on par with the number of backed | ||
| // candidates. | ||
| assert_finalized_block_height(¶_client, 12..16).await?; | ||
|
|
||
| log::info!("Test finished successfully"); | ||
|
|
||
| Ok(()) | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,6 @@ | ||
| // Copyright (C) Parity Technologies (UK) Ltd. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| mod basic_3cores; | ||
| mod doesnt_break_parachains; | ||
| mod slot_based_3cores; |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
These ranges don't make sense to me. 40 > 3 *12 .. are we too strict for the elastic scaling case or too lenient for the non-elastic scaling case?
Also note: I absolutely had to lookup the code of
assert_para_throughputto understand what this is doing. Not a big deal, but e.g. using types like ParaId would have made it easier.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So these ranges mean that:
we are waiting until we reached 15 finalized blocks (after the first session change, and don't count blocks with session changes).
Then, check that the number of backed blocks for para 2000 is within the 40..46 range (ideally it should be 45, since it's 3*15, but in reality the performance in the CI is not as good).
And check that the number of backed blocks for para 2001 is within the 12..16 range (this para only has one assigned core, so it can get at most 15 blocks in, again allow some buffer here for less performant hardware)
I can do that 👍🏻