From 7d04fbc81e15814705bb85cf2d1115d69341eae6 Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Thu, 8 Aug 2024 10:04:00 +0200 Subject: [PATCH] Add offchain executor params --- Cargo.lock | 1 + .../chain_spec_runtime/src/presets.rs | 2 ++ .../bin/utils/chain-spec-builder/bin/main.rs | 4 +-- .../bin/utils/chain-spec-builder/src/lib.rs | 7 ++-- .../client/allocator/src/freeing_bump.rs | 6 ++-- substrate/client/chain-spec/src/chain_spec.rs | 32 ++++++++++++++++--- .../chain-spec/src/genesis_config_builder.rs | 13 +++++--- substrate/client/chain-spec/src/lib.rs | 10 ++++++ substrate/client/cli/Cargo.toml | 1 + .../client/cli/src/commands/build_spec_cmd.rs | 7 ++++ substrate/primitives/core/src/lib.rs | 4 +-- 11 files changed, 69 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f41f3ca1eb2..f78c99d047fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17134,6 +17134,7 @@ dependencies = [ "rand", "regex", "rpassword", + "sc-chain-spec", "sc-client-api", "sc-client-db", "sc-keystore", diff --git a/docs/sdk/src/reference_docs/chain_spec_runtime/src/presets.rs b/docs/sdk/src/reference_docs/chain_spec_runtime/src/presets.rs index 02c2d90f7c82..406ff12cbb84 100644 --- a/docs/sdk/src/reference_docs/chain_spec_runtime/src/presets.rs +++ b/docs/sdk/src/reference_docs/chain_spec_runtime/src/presets.rs @@ -144,6 +144,7 @@ pub fn get_builtin_preset(id: &sp_genesis_builder::PresetId) -> Option::new( crate::WASM_BINARY.expect("wasm binary shall exists"), + Default::default(), ); assert!(builder.get_storage_for_named_preset(Some(&PRESET_1.to_string())).is_ok()); assert!(builder.get_storage_for_named_preset(Some(&PRESET_2.to_string())).is_ok()); @@ -156,6 +157,7 @@ fn check_presets() { fn invalid_preset_works() { let builder = sc_chain_spec::GenesisConfigBuilderRuntimeCaller::<()>::new( crate::WASM_BINARY.expect("wasm binary shall exists"), + Default::default(), ); // Even though a preset contains invalid_key, conversion to raw storage does not fail. This is // because the [`FooStruct`] structure is not annotated with `deny_unknown_fields` [`serde`] diff --git a/substrate/bin/utils/chain-spec-builder/bin/main.rs b/substrate/bin/utils/chain-spec-builder/bin/main.rs index 39fa054b4806..27d53f876320 100644 --- a/substrate/bin/utils/chain-spec-builder/bin/main.rs +++ b/substrate/bin/utils/chain-spec-builder/bin/main.rs @@ -109,7 +109,7 @@ fn inner_main() -> Result<(), String> { let code = fs::read(runtime_wasm_path.as_path()) .map_err(|e| format!("wasm blob shall be readable {e}"))?; let caller: GenesisConfigBuilderRuntimeCaller = - GenesisConfigBuilderRuntimeCaller::new(&code[..]); + GenesisConfigBuilderRuntimeCaller::new(&code[..], Default::default()); let presets = caller .preset_names() .map_err(|e| format!("getting default config from runtime should work: {e}"))?; @@ -129,7 +129,7 @@ fn inner_main() -> Result<(), String> { let code = fs::read(runtime_wasm_path.as_path()) .map_err(|e| format!("wasm blob shall be readable {e}"))?; let caller: GenesisConfigBuilderRuntimeCaller = - GenesisConfigBuilderRuntimeCaller::new(&code[..]); + GenesisConfigBuilderRuntimeCaller::new(&code[..], Default::default()); let preset = caller .get_named_preset(preset_name.as_ref()) .map_err(|e| format!("getting default config from runtime should work: {e}"))?; diff --git a/substrate/bin/utils/chain-spec-builder/src/lib.rs b/substrate/bin/utils/chain-spec-builder/src/lib.rs index 6c679f109a00..36066eaaef33 100644 --- a/substrate/bin/utils/chain-spec-builder/src/lib.rs +++ b/substrate/bin/utils/chain-spec-builder/src/lib.rs @@ -120,7 +120,7 @@ use std::{fs, path::PathBuf}; use clap::{Parser, Subcommand}; -use sc_chain_spec::{ChainType, GenericChainSpec, GenesisConfigBuilderRuntimeCaller}; +use sc_chain_spec::{OffchainExecutorParams, ChainType, GenericChainSpec, GenesisConfigBuilderRuntimeCaller}; use serde_json::Value; /// A utility to easily create a chain spec definition. @@ -168,6 +168,9 @@ pub struct CreateCmd { /// errors will be reported. #[arg(long, short = 'v')] verify: bool, + /// Number of extra heap pages available to executor when calling into runtime to build chain spec. + #[arg(long)] + extra_heap_pages: Option, #[command(subcommand)] action: GenesisBuildAction, } @@ -310,7 +313,7 @@ pub fn generate_chain_spec_for_runtime(cmd: &CreateCmd) -> Result { let caller: GenesisConfigBuilderRuntimeCaller = - GenesisConfigBuilderRuntimeCaller::new(&code[..]); + GenesisConfigBuilderRuntimeCaller::new(&code[..], OffchainExecutorParams { extra_heap_pages: cmd.extra_heap_pages }); let default_config = caller .get_default_config() .map_err(|e| format!("getting default config from runtime should work: {e}"))?; diff --git a/substrate/client/allocator/src/freeing_bump.rs b/substrate/client/allocator/src/freeing_bump.rs index 144c0764540d..f977df7faacb 100644 --- a/substrate/client/allocator/src/freeing_bump.rs +++ b/substrate/client/allocator/src/freeing_bump.rs @@ -96,13 +96,13 @@ const LOG_TARGET: &str = "wasm-heap"; // The minimum possible allocation size is chosen to be 8 bytes because in that case we would have // easier time to provide the guaranteed alignment of 8. // -// The maximum possible allocation size is set in the primitives to 32MiB. +// The maximum possible allocation size is set in the primitives to 64MiB. // // N_ORDERS - represents the number of orders supported. // // This number corresponds to the number of powers between the minimum possible allocation and -// maximum possible allocation, or: 2^3...2^25 (both ends inclusive, hence 23). -const N_ORDERS: usize = 23; +// maximum possible allocation, or: 2^3...2^26 (both ends inclusive, hence 24). +const N_ORDERS: usize = 24; const MIN_POSSIBLE_ALLOCATION: u32 = 8; // 2^3 bytes, 8 bytes /// The exponent for the power of two sized block adjusted to the minimum size. diff --git a/substrate/client/chain-spec/src/chain_spec.rs b/substrate/client/chain-spec/src/chain_spec.rs index aa3c1ba3e6f1..1a2b6952ddf1 100644 --- a/substrate/client/chain-spec/src/chain_spec.rs +++ b/substrate/client/chain-spec/src/chain_spec.rs @@ -18,6 +18,7 @@ //! Substrate chain configurations. #![warn(missing_docs)] +use crate::OffchainExecutorParams; use crate::{ extension::GetExtension, genesis_config_builder::HostFunctions, ChainType, GenesisConfigBuilderRuntimeCaller as RuntimeCaller, Properties, @@ -121,7 +122,7 @@ impl GenesisSource { code: code.clone(), })), Self::GenesisBuilderApi(GenesisBuildAction::NamedPreset(name, _), code) => { - let patch = RuntimeCaller::::new(&code[..]).get_named_preset(Some(name))?; + let patch = RuntimeCaller::::new(&code[..], Default::default()).get_named_preset(Some(name))?; Ok(Genesis::RuntimeGenesis(RuntimeGenesisInner { json_blob: RuntimeGenesisConfigJson::Patch(patch), code: code.clone(), @@ -158,7 +159,7 @@ where json_blob: RuntimeGenesisConfigJson::Config(config), code, }) => { - RuntimeCaller::::new(&code[..]) + RuntimeCaller::::new(&code[..], self.executor_params.clone()) .get_storage_for_config(config)? .assimilate_storage(storage)?; storage @@ -169,7 +170,7 @@ where json_blob: RuntimeGenesisConfigJson::Patch(patch), code, }) => { - RuntimeCaller::::new(&code[..]) + RuntimeCaller::::new(&code[..], self.executor_params.clone()) .get_storage_for_patch(patch)? .assimilate_storage(storage)?; storage @@ -314,6 +315,7 @@ pub struct ChainSpecBuilder { protocol_id: Option, fork_id: Option, properties: Option, + heap_pages: Option, } impl ChainSpecBuilder { @@ -331,6 +333,7 @@ impl ChainSpecBuilder { protocol_id: None, fork_id: None, properties: None, + heap_pages: None, } } @@ -413,6 +416,12 @@ impl ChainSpecBuilder { self } + /// Sets the number of heap pages available to the executor during chain spec building. + pub fn with_heap_pages(mut self, pages: u64) -> Self { + self.heap_pages = Some(pages); + self + } + /// Builds a [`ChainSpec`] instance using the provided settings. pub fn build(self) -> ChainSpec { let client_spec = ClientSpec { @@ -430,9 +439,14 @@ impl ChainSpecBuilder { code_substitutes: BTreeMap::new(), }; + let executor_params = OffchainExecutorParams { + extra_heap_pages: self.heap_pages, + }; + ChainSpec { client_spec, genesis: GenesisSource::GenesisBuilderApi(self.genesis_build_action, self.code.into()), + executor_params, _host_functions: Default::default(), } } @@ -446,6 +460,7 @@ impl ChainSpecBuilder { pub struct ChainSpec { client_spec: ClientSpec, genesis: GenesisSource, + executor_params: OffchainExecutorParams, _host_functions: PhantomData, } @@ -454,6 +469,7 @@ impl Clone for ChainSpec { ChainSpec { client_spec: self.client_spec.clone(), genesis: self.genesis.clone(), + executor_params: self.executor_params.clone(), _host_functions: self._host_functions, } } @@ -533,6 +549,7 @@ impl ChainSpec { Ok(ChainSpec { client_spec, genesis: GenesisSource::Binary(json), + executor_params: Default::default(), _host_functions: Default::default(), }) } @@ -556,6 +573,7 @@ impl ChainSpec { Ok(ChainSpec { client_spec, genesis: GenesisSource::File(path), + executor_params: Default::default(), _host_functions: Default::default(), }) } @@ -586,7 +604,7 @@ where }), ) => { let mut storage = - RuntimeCaller::::new(&code[..]).get_storage_for_config(config)?; + RuntimeCaller::::new(&code[..], self.executor_params.clone()).get_storage_for_config(config)?; storage.top.insert(sp_core::storage::well_known_keys::CODE.to_vec(), code); RawGenesis::from(storage) }, @@ -598,7 +616,7 @@ where }), ) => { let mut storage = - RuntimeCaller::::new(&code[..]).get_storage_for_patch(patch)?; + RuntimeCaller::::new(&code[..], self.executor_params.clone()).get_storage_for_patch(patch)?; storage.top.insert(sp_core::storage::well_known_keys::CODE.to_vec(), code); RawGenesis::from(storage) }, @@ -692,6 +710,10 @@ where .map(|(h, c)| (h.clone(), c.0.clone())) .collect() } + + fn set_executor_params(&mut self, executor_params: OffchainExecutorParams) { + self.executor_params = executor_params; + } } /// The `fun` will be called with the value at `path`. diff --git a/substrate/client/chain-spec/src/genesis_config_builder.rs b/substrate/client/chain-spec/src/genesis_config_builder.rs index 13a2f3c072f5..2e6b9cfb4e7d 100644 --- a/substrate/client/chain-spec/src/genesis_config_builder.rs +++ b/substrate/client/chain-spec/src/genesis_config_builder.rs @@ -30,6 +30,8 @@ use sp_genesis_builder::{PresetId, Result as BuildResult}; use sp_state_machine::BasicExternalities; use std::borrow::Cow; +use crate::OffchainExecutorParams; + /// A utility that facilitates calling the GenesisBuilder API from the runtime wasm code blob. /// /// `EHF` type allows to specify the extended host function required for building runtime's genesis @@ -59,13 +61,16 @@ where /// Creates new instance using the provided code blob. /// /// This code is later referred to as `runtime`. - pub fn new(code: &'a [u8]) -> Self { + pub fn new(code: &'a [u8], executor_params: OffchainExecutorParams) -> Self { + let mut executor = WasmExecutor::<(sp_io::SubstrateHostFunctions, EHF)>::builder() + .with_allow_missing_host_functions(true); + if let Some(heap_pages) = executor_params.extra_heap_pages { + executor = executor.with_offchain_heap_alloc_strategy(sc_executor::HeapAllocStrategy::Static { extra_pages: heap_pages as _ }) + }; GenesisConfigBuilderRuntimeCaller { code: code.into(), code_hash: sp_crypto_hashing::blake2_256(code).to_vec(), - executor: WasmExecutor::<(sp_io::SubstrateHostFunctions, EHF)>::builder() - .with_allow_missing_host_functions(true) - .build(), + executor: executor.build(), } } diff --git a/substrate/client/chain-spec/src/lib.rs b/substrate/client/chain-spec/src/lib.rs index c43f9e89b8a9..f0862174a4e4 100644 --- a/substrate/client/chain-spec/src/lib.rs +++ b/substrate/client/chain-spec/src/lib.rs @@ -369,6 +369,14 @@ pub enum ChainType { Custom(String), } +/// Executor parameters used to call into the runtime when building a chain spec. +#[derive(Default, Clone)] +pub struct OffchainExecutorParams { + /// Number of extra heap pages available to the executor. If not specified, executor's + /// default is used. + pub extra_heap_pages: Option, +} + impl Default for ChainType { fn default() -> Self { Self::Live @@ -416,6 +424,8 @@ pub trait ChainSpec: BuildStorage + Send + Sync { fn set_storage(&mut self, storage: Storage); /// Returns code substitutes that should be used for the on chain wasm. fn code_substitutes(&self) -> std::collections::BTreeMap>; + /// Sets executor parameters that should be used when building this chain spec. + fn set_executor_params(&mut self, executor_params: OffchainExecutorParams); } impl std::fmt::Debug for dyn ChainSpec { diff --git a/substrate/client/cli/Cargo.toml b/substrate/client/cli/Cargo.toml index b7d29aebc3d7..b627ee9e190c 100644 --- a/substrate/client/cli/Cargo.toml +++ b/substrate/client/cli/Cargo.toml @@ -35,6 +35,7 @@ thiserror = { workspace = true } # personal fork here as workaround for: https://github.com/rust-bitcoin/rust-bip39/pull/64 bip39 = { package = "parity-bip39", version = "2.0.1", features = ["rand"] } tokio = { features = ["parking_lot", "rt-multi-thread", "signal"], workspace = true, default-features = true } +sc-chain-spec = { workspace = true, default-features = true } sc-client-api = { workspace = true, default-features = true } sc-client-db = { workspace = true } sc-keystore = { workspace = true, default-features = true } diff --git a/substrate/client/cli/src/commands/build_spec_cmd.rs b/substrate/client/cli/src/commands/build_spec_cmd.rs index df8c6b7d0baa..0f1d91cce8c7 100644 --- a/substrate/client/cli/src/commands/build_spec_cmd.rs +++ b/substrate/client/cli/src/commands/build_spec_cmd.rs @@ -23,6 +23,7 @@ use crate::{ }; use clap::Parser; use log::info; +use sc_chain_spec::OffchainExecutorParams; use sc_network::config::build_multiaddr; use sc_service::{ config::{MultiaddrWithPeerId, NetworkConfiguration}, @@ -43,6 +44,10 @@ pub struct BuildSpecCmd { #[arg(long)] pub disable_default_bootnode: bool, + /// Number of extra heap pages available to genesis builder. + #[arg(long)] + pub extra_heap_pages: Option, + #[allow(missing_docs)] #[clap(flatten)] pub shared_params: SharedParams, @@ -72,6 +77,8 @@ impl BuildSpecCmd { spec.add_boot_node(addr) } + spec.set_executor_params(OffchainExecutorParams { extra_heap_pages: self.extra_heap_pages }); + let json = sc_service::chain_ops::build_spec(&*spec, raw_output)?; if std::io::stdout().write_all(json.as_bytes()).is_err() { let _ = std::io::stderr().write_all(b"Error writing to stdout\n"); diff --git a/substrate/primitives/core/src/lib.rs b/substrate/primitives/core/src/lib.rs index bb05bebc6274..7151a9bb0c3c 100644 --- a/substrate/primitives/core/src/lib.rs +++ b/substrate/primitives/core/src/lib.rs @@ -406,9 +406,9 @@ macro_rules! impl_maybe_marker_std_or_serde { } /// The maximum number of bytes that can be allocated at one time. -// The maximum possible allocation size was chosen rather arbitrary, 32 MiB should be enough for +// The maximum possible allocation size was chosen rather arbitrary, 64 MiB should be enough for // everybody. -pub const MAX_POSSIBLE_ALLOCATION: u32 = 33554432; // 2^25 bytes, 32 MiB +pub const MAX_POSSIBLE_ALLOCATION: u32 = 67108864; // 2^26 bytes, 64 MiB /// Generates a macro for checking if a certain feature is enabled. ///