Add offchain executor params

docs/sdk/src/reference_docs/chain_spec_runtime/src/presets.rs

@@ -144,6 +144,7 @@ pub fn get_builtin_preset(id: &sp_genesis_builder::PresetId) -> Option<alloc::ve
 fn check_presets() {
 	let builder = sc_chain_spec::GenesisConfigBuilderRuntimeCaller::<()>::new(
 		crate::WASM_BINARY.expect("wasm binary shall exists"),
+		Default::default(),
 	);
 	assert!(builder.get_storage_for_named_preset(Some(&PRESET_1.to_string())).is_ok());
 	assert!(builder.get_storage_for_named_preset(Some(&PRESET_2.to_string())).is_ok());
@@ -156,6 +157,7 @@ fn check_presets() {
 fn invalid_preset_works() {
 	let builder = sc_chain_spec::GenesisConfigBuilderRuntimeCaller::<()>::new(
 		crate::WASM_BINARY.expect("wasm binary shall exists"),
+		Default::default(),
 	);
 	// Even though a preset contains invalid_key, conversion to raw storage does not fail. This is
 	// because the [`FooStruct`] structure is not annotated with `deny_unknown_fields` [`serde`]

substrate/bin/utils/chain-spec-builder/bin/main.rs

@@ -109,7 +109,7 @@ fn inner_main() -> Result<(), String> {
 			let code = fs::read(runtime_wasm_path.as_path())
 				.map_err(|e| format!("wasm blob shall be readable {e}"))?;
 			let caller: GenesisConfigBuilderRuntimeCaller =
-				GenesisConfigBuilderRuntimeCaller::new(&code[..]);
+				GenesisConfigBuilderRuntimeCaller::new(&code[..], Default::default());
 			let presets = caller
 				.preset_names()
 				.map_err(|e| format!("getting default config from runtime should work: {e}"))?;
@@ -129,7 +129,7 @@ fn inner_main() -> Result<(), String> {
 			let code = fs::read(runtime_wasm_path.as_path())
 				.map_err(|e| format!("wasm blob shall be readable {e}"))?;
 			let caller: GenesisConfigBuilderRuntimeCaller =
-				GenesisConfigBuilderRuntimeCaller::new(&code[..]);
+				GenesisConfigBuilderRuntimeCaller::new(&code[..], Default::default());
 			let preset = caller
 				.get_named_preset(preset_name.as_ref())
 				.map_err(|e| format!("getting default config from runtime should work: {e}"))?;

substrate/bin/utils/chain-spec-builder/src/lib.rs

@@ -120,7 +120,7 @@
 use std::{fs, path::PathBuf};
 
 use clap::{Parser, Subcommand};
-use sc_chain_spec::{ChainType, GenericChainSpec, GenesisConfigBuilderRuntimeCaller};
+use sc_chain_spec::{OffchainExecutorParams, ChainType, GenericChainSpec, GenesisConfigBuilderRuntimeCaller};
 use serde_json::Value;
 
 /// A utility to easily create a chain spec definition.
@@ -168,6 +168,9 @@ pub struct CreateCmd {
 	/// errors will be reported.
 	#[arg(long, short = 'v')]
 	verify: bool,
+	/// Number of extra heap pages available to executor when calling into runtime to build chain spec.
+	#[arg(long)]
+	extra_heap_pages: Option<u64>,
 	#[command(subcommand)]
 	action: GenesisBuildAction,
 }
@@ -310,7 +313,7 @@ pub fn generate_chain_spec_for_runtime(cmd: &CreateCmd) -> Result<String, String
 		},
 		GenesisBuildAction::Default(DefaultCmd {}) => {
 			let caller: GenesisConfigBuilderRuntimeCaller =
-				GenesisConfigBuilderRuntimeCaller::new(&code[..]);
+				GenesisConfigBuilderRuntimeCaller::new(&code[..], OffchainExecutorParams { extra_heap_pages: cmd.extra_heap_pages });
 			let default_config = caller
 				.get_default_config()
 				.map_err(|e| format!("getting default config from runtime should work: {e}"))?;

substrate/client/allocator/src/freeing_bump.rs

@@ -96,13 +96,13 @@ const LOG_TARGET: &str = "wasm-heap";
 // The minimum possible allocation size is chosen to be 8 bytes because in that case we would have
 // easier time to provide the guaranteed alignment of 8.
 //
-// The maximum possible allocation size is set in the primitives to 32MiB.
+// The maximum possible allocation size is set in the primitives to 64MiB.
 //
 // N_ORDERS - represents the number of orders supported.
 //
 // This number corresponds to the number of powers between the minimum possible allocation and
-// maximum possible allocation, or: 2^3...2^25 (both ends inclusive, hence 23).
-const N_ORDERS: usize = 23;
+// maximum possible allocation, or: 2^3...2^26 (both ends inclusive, hence 24).
+const N_ORDERS: usize = 24;
 const MIN_POSSIBLE_ALLOCATION: u32 = 8; // 2^3 bytes, 8 bytes
 
 /// The exponent for the power of two sized block adjusted to the minimum size.

substrate/client/chain-spec/src/chain_spec.rs

@@ -18,6 +18,7 @@
 
 //! Substrate chain configurations.
 #![warn(missing_docs)]
+use crate::OffchainExecutorParams;
 use crate::{
 	extension::GetExtension, genesis_config_builder::HostFunctions, ChainType,
 	GenesisConfigBuilderRuntimeCaller as RuntimeCaller, Properties,
@@ -121,7 +122,7 @@ impl<EHF: HostFunctions> GenesisSource<EHF> {
 					code: code.clone(),
 				})),
 			Self::GenesisBuilderApi(GenesisBuildAction::NamedPreset(name, _), code) => {
-				let patch = RuntimeCaller::<EHF>::new(&code[..]).get_named_preset(Some(name))?;
+				let patch = RuntimeCaller::<EHF>::new(&code[..], Default::default()).get_named_preset(Some(name))?;
 				Ok(Genesis::RuntimeGenesis(RuntimeGenesisInner {
 					json_blob: RuntimeGenesisConfigJson::Patch(patch),
 					code: code.clone(),
@@ -158,7 +159,7 @@ where
 				json_blob: RuntimeGenesisConfigJson::Config(config),
 				code,
 			}) => {
-				RuntimeCaller::<EHF>::new(&code[..])
+				RuntimeCaller::<EHF>::new(&code[..], self.executor_params.clone())
 					.get_storage_for_config(config)?
 					.assimilate_storage(storage)?;
 				storage
@@ -169,7 +170,7 @@ where
 				json_blob: RuntimeGenesisConfigJson::Patch(patch),
 				code,
 			}) => {
-				RuntimeCaller::<EHF>::new(&code[..])
+				RuntimeCaller::<EHF>::new(&code[..], self.executor_params.clone())
 					.get_storage_for_patch(patch)?
 					.assimilate_storage(storage)?;
 				storage
@@ -314,6 +315,7 @@ pub struct ChainSpecBuilder<E = NoExtension, EHF = ()> {
 	protocol_id: Option<String>,
 	fork_id: Option<String>,
 	properties: Option<Properties>,
+	heap_pages: Option<u64>,
 }
 
 impl<E, EHF> ChainSpecBuilder<E, EHF> {
@@ -331,6 +333,7 @@ impl<E, EHF> ChainSpecBuilder<E, EHF> {
 			protocol_id: None,
 			fork_id: None,
 			properties: None,
+			heap_pages: None,
 		}
 	}
 
@@ -413,6 +416,12 @@ impl<E, EHF> ChainSpecBuilder<E, EHF> {
 		self
 	}
 
+	/// Sets the number of heap pages available to the executor during chain spec building.
+	pub fn with_heap_pages(mut self, pages: u64) -> Self {
+		self.heap_pages = Some(pages);
+		self
+	}
+
 	/// Builds a [`ChainSpec`] instance using the provided settings.
 	pub fn build(self) -> ChainSpec<E, EHF> {
 		let client_spec = ClientSpec {
@@ -430,9 +439,14 @@ impl<E, EHF> ChainSpecBuilder<E, EHF> {
 			code_substitutes: BTreeMap::new(),
 		};
 
+		let executor_params = OffchainExecutorParams {
+			extra_heap_pages: self.heap_pages,
+		};
+
 		ChainSpec {
 			client_spec,
 			genesis: GenesisSource::GenesisBuilderApi(self.genesis_build_action, self.code.into()),
+			executor_params,
 			_host_functions: Default::default(),
 		}
 	}
@@ -446,6 +460,7 @@ impl<E, EHF> ChainSpecBuilder<E, EHF> {
 pub struct ChainSpec<E = NoExtension, EHF = ()> {
 	client_spec: ClientSpec<E>,
 	genesis: GenesisSource<EHF>,
+	executor_params: OffchainExecutorParams,
 	_host_functions: PhantomData<EHF>,
 }
 
@@ -454,6 +469,7 @@ impl<E: Clone, EHF> Clone for ChainSpec<E, EHF> {
 		ChainSpec {
 			client_spec: self.client_spec.clone(),
 			genesis: self.genesis.clone(),
+			executor_params: self.executor_params.clone(),
 			_host_functions: self._host_functions,
 		}
 	}
@@ -533,6 +549,7 @@ impl<E: serde::de::DeserializeOwned, EHF> ChainSpec<E, EHF> {
 		Ok(ChainSpec {
 			client_spec,
 			genesis: GenesisSource::Binary(json),
+			executor_params: Default::default(),
 			_host_functions: Default::default(),
 		})
 	}
@@ -556,6 +573,7 @@ impl<E: serde::de::DeserializeOwned, EHF> ChainSpec<E, EHF> {
 		Ok(ChainSpec {
 			client_spec,
 			genesis: GenesisSource::File(path),
+			executor_params: Default::default(),
 			_host_functions: Default::default(),
 		})
 	}
@@ -586,7 +604,7 @@ where
 				}),
 			) => {
 				let mut storage =
-					RuntimeCaller::<EHF>::new(&code[..]).get_storage_for_config(config)?;
+					RuntimeCaller::<EHF>::new(&code[..], self.executor_params.clone()).get_storage_for_config(config)?;
 				storage.top.insert(sp_core::storage::well_known_keys::CODE.to_vec(), code);
 				RawGenesis::from(storage)
 			},
@@ -598,7 +616,7 @@ where
 				}),
 			) => {
 				let mut storage =
-					RuntimeCaller::<EHF>::new(&code[..]).get_storage_for_patch(patch)?;
+					RuntimeCaller::<EHF>::new(&code[..], self.executor_params.clone()).get_storage_for_patch(patch)?;
 				storage.top.insert(sp_core::storage::well_known_keys::CODE.to_vec(), code);
 				RawGenesis::from(storage)
 			},
@@ -692,6 +710,10 @@ where
 			.map(|(h, c)| (h.clone(), c.0.clone()))
 			.collect()
 	}
+
+	fn set_executor_params(&mut self, executor_params: OffchainExecutorParams) {
+		self.executor_params = executor_params;
+	}
 }
 
 /// The `fun` will be called with the value at `path`.

substrate/client/chain-spec/src/genesis_config_builder.rs

@@ -30,6 +30,8 @@ use sp_genesis_builder::{PresetId, Result as BuildResult};
 use sp_state_machine::BasicExternalities;
 use std::borrow::Cow;
 
+use crate::OffchainExecutorParams;
+
 /// A utility that facilitates calling the GenesisBuilder API from the runtime wasm code blob.
 ///
 /// `EHF` type allows to specify the extended host function required for building runtime's genesis
@@ -59,13 +61,16 @@ where
 	/// Creates new instance using the provided code blob.
 	///
 	/// This code is later referred to as `runtime`.
-	pub fn new(code: &'a [u8]) -> Self {
+	pub fn new(code: &'a [u8], executor_params: OffchainExecutorParams) -> Self {
+		let mut executor = WasmExecutor::<(sp_io::SubstrateHostFunctions, EHF)>::builder()
+				.with_allow_missing_host_functions(true);
+		if let Some(heap_pages) = executor_params.extra_heap_pages {
+			executor = executor.with_offchain_heap_alloc_strategy(sc_executor::HeapAllocStrategy::Static { extra_pages: heap_pages as _ })
+		};
 		GenesisConfigBuilderRuntimeCaller {
 			code: code.into(),
 			code_hash: sp_crypto_hashing::blake2_256(code).to_vec(),
-			executor: WasmExecutor::<(sp_io::SubstrateHostFunctions, EHF)>::builder()
-				.with_allow_missing_host_functions(true)
-				.build(),
+			executor: executor.build(),
 		}
 	}
 

substrate/client/chain-spec/src/lib.rs

@@ -369,6 +369,14 @@ pub enum ChainType {
 	Custom(String),
 }
 
+/// Executor parameters used to call into the runtime when building a chain spec.
+#[derive(Default, Clone)]
+pub struct OffchainExecutorParams {
+	/// Number of extra heap pages available to the executor. If not specified, executor's
+	/// default is used.
+	pub extra_heap_pages: Option<u64>,
+}
+
 impl Default for ChainType {
 	fn default() -> Self {
 		Self::Live
@@ -416,6 +424,8 @@ pub trait ChainSpec: BuildStorage + Send + Sync {
 	fn set_storage(&mut self, storage: Storage);
 	/// Returns code substitutes that should be used for the on chain wasm.
 	fn code_substitutes(&self) -> std::collections::BTreeMap<String, Vec<u8>>;
+	/// Sets executor parameters that should be used when building this chain spec.
+	fn set_executor_params(&mut self, executor_params: OffchainExecutorParams);
 }
 
 impl std::fmt::Debug for dyn ChainSpec {

substrate/client/cli/Cargo.toml

@@ -35,6 +35,7 @@ thiserror = { workspace = true }
 # personal fork here as workaround for: https://github.com/rust-bitcoin/rust-bip39/pull/64
 bip39 = { package = "parity-bip39", version = "2.0.1", features = ["rand"] }
 tokio = { features = ["parking_lot", "rt-multi-thread", "signal"], workspace = true, default-features = true }
+sc-chain-spec = { workspace = true, default-features = true }
 sc-client-api = { workspace = true, default-features = true }
 sc-client-db = { workspace = true }
 sc-keystore = { workspace = true, default-features = true }

substrate/client/cli/src/commands/build_spec_cmd.rs

@@ -23,6 +23,7 @@ use crate::{
 };
 use clap::Parser;
 use log::info;
+use sc_chain_spec::OffchainExecutorParams;
 use sc_network::config::build_multiaddr;
 use sc_service::{
 	config::{MultiaddrWithPeerId, NetworkConfiguration},
@@ -43,6 +44,10 @@ pub struct BuildSpecCmd {
 	#[arg(long)]
 	pub disable_default_bootnode: bool,
 
+	/// Number of extra heap pages available to genesis builder.
+	#[arg(long)]
+	pub extra_heap_pages: Option<u64>,
+
 	#[allow(missing_docs)]
 	#[clap(flatten)]
 	pub shared_params: SharedParams,
@@ -72,6 +77,8 @@ impl BuildSpecCmd {
 			spec.add_boot_node(addr)
 		}
 
+		spec.set_executor_params(OffchainExecutorParams { extra_heap_pages: self.extra_heap_pages });
+
 		let json = sc_service::chain_ops::build_spec(&*spec, raw_output)?;
 		if std::io::stdout().write_all(json.as_bytes()).is_err() {
 			let _ = std::io::stderr().write_all(b"Error writing to stdout\n");

substrate/primitives/core/src/lib.rs

@@ -406,9 +406,9 @@ macro_rules! impl_maybe_marker_std_or_serde {
 }
 
 /// The maximum number of bytes that can be allocated at one time.
-// The maximum possible allocation size was chosen rather arbitrary, 32 MiB should be enough for
+// The maximum possible allocation size was chosen rather arbitrary, 64 MiB should be enough for
 // everybody.
-pub const MAX_POSSIBLE_ALLOCATION: u32 = 33554432; // 2^25 bytes, 32 MiB
+pub const MAX_POSSIBLE_ALLOCATION: u32 = 67108864; // 2^26 bytes, 64 MiB
 
 /// Generates a macro for checking if a certain feature is enabled.
 ///