[DNM] Add offchain executor params

cumulus/test/service/src/chain_spec.rs

@@ -50,7 +50,8 @@ pub fn get_chain_spec_with_extra_endowed(
 	extra_endowed_accounts: Vec<AccountId>,
 	code: &[u8],
 ) -> ChainSpec {
-	let runtime_caller = GenesisConfigBuilderRuntimeCaller::<ParachainHostFunctions>::new(code);
+	let runtime_caller =
+		GenesisConfigBuilderRuntimeCaller::<ParachainHostFunctions>::new(code, Default::default());
 	let mut development_preset = runtime_caller
 		.get_named_preset(Some(&sp_genesis_builder::LOCAL_TESTNET_RUNTIME_PRESET.to_string()))
 		.expect("development preset is available on test runtime; qed");

docs/sdk/src/reference_docs/chain_spec_runtime/src/presets.rs

@@ -142,6 +142,7 @@ pub fn get_builtin_preset(id: &sp_genesis_builder::PresetId) -> Option<alloc::ve
 fn check_presets() {
 	let builder = sc_chain_spec::GenesisConfigBuilderRuntimeCaller::<()>::new(
 		crate::WASM_BINARY.expect("wasm binary shall exists"),
+		Default::default(),
 	);
 	assert!(builder.get_storage_for_named_preset(Some(&PRESET_1.to_string())).is_ok());
 	assert!(builder.get_storage_for_named_preset(Some(&PRESET_2.to_string())).is_ok());
@@ -154,6 +155,7 @@ fn check_presets() {
 fn invalid_preset_works() {
 	let builder = sc_chain_spec::GenesisConfigBuilderRuntimeCaller::<()>::new(
 		crate::WASM_BINARY.expect("wasm binary shall exists"),
+		Default::default(),
 	);
 	// Even though a preset contains invalid_key, conversion to raw storage does not fail. This is
 	// because the [`FooStruct`] structure is not annotated with `deny_unknown_fields` [`serde`]

substrate/bin/utils/chain-spec-builder/src/lib.rs

@@ -22,7 +22,7 @@ docify::compile_markdown!("README.docify.md", "README.md");
 use clap::{Parser, Subcommand};
 use sc_chain_spec::{
 	json_patch, set_code_substitute_in_json_chain_spec, update_code_in_json_chain_spec, ChainType,
-	GenericChainSpec, GenesisConfigBuilderRuntimeCaller,
+	GenericChainSpec, GenesisConfigBuilderRuntimeCaller, OffchainExecutorParams,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -83,6 +83,10 @@ pub struct CreateCmd {
 	/// errors will be reported.
 	#[arg(long, short = 'v')]
 	verify: bool,
+	/// Number of extra heap pages available to executor when calling into runtime to build chain
+	/// spec.
+	#[arg(long)]
+	extra_heap_pages: Option<u64>,
 	#[command(subcommand)]
 	action: GenesisBuildAction,
 
@@ -289,7 +293,7 @@ impl ChainSpecBuilder {
 				let code = fs::read(runtime.as_path())
 					.map_err(|e| format!("wasm blob shall be readable {e}"))?;
 				let caller: GenesisConfigBuilderRuntimeCaller =
-					GenesisConfigBuilderRuntimeCaller::new(&code[..]);
+					GenesisConfigBuilderRuntimeCaller::new(&code[..], Default::default());
 				let presets = caller
 					.preset_names()
 					.map_err(|e| format!("getting default config from runtime should work: {e}"))?;
@@ -299,7 +303,7 @@ impl ChainSpecBuilder {
 				let code = fs::read(runtime.as_path())
 					.map_err(|e| format!("wasm blob shall be readable {e}"))?;
 				let caller: GenesisConfigBuilderRuntimeCaller =
-					GenesisConfigBuilderRuntimeCaller::new(&code[..]);
+					GenesisConfigBuilderRuntimeCaller::new(&code[..], Default::default());
 				let preset = caller
 					.get_named_preset(preset_name.as_ref())
 					.map_err(|e| format!("getting default config from runtime should work: {e}"))?;
@@ -348,8 +352,10 @@ fn process_action<T: Serialize + Clone + Sync + 'static>(
 			)?)
 		},
 		GenesisBuildAction::Default(DefaultCmd {}) => {
-			let caller: GenesisConfigBuilderRuntimeCaller =
-				GenesisConfigBuilderRuntimeCaller::new(&code);
+			let caller: GenesisConfigBuilderRuntimeCaller = GenesisConfigBuilderRuntimeCaller::new(
+				&code,
+				OffchainExecutorParams { extra_heap_pages: cmd.extra_heap_pages },
+			);
 			let default_config = caller
 				.get_default_config()
 				.map_err(|e| format!("getting default config from runtime should work: {e}"))?;

substrate/client/chain-spec/src/chain_spec.rs

@@ -20,7 +20,7 @@
 #![warn(missing_docs)]
 use crate::{
 	extension::GetExtension, genesis_config_builder::HostFunctions, ChainType,
-	GenesisConfigBuilderRuntimeCaller as RuntimeCaller, Properties,
+	GenesisConfigBuilderRuntimeCaller as RuntimeCaller, OffchainExecutorParams, Properties,
 };
 use sc_network::config::MultiaddrWithPeerId;
 use sc_telemetry::TelemetryEndpoints;
@@ -121,7 +121,8 @@ impl<EHF: HostFunctions> GenesisSource<EHF> {
 					code: code.clone(),
 				})),
 			Self::GenesisBuilderApi(GenesisBuildAction::NamedPreset(name, _), code) => {
-				let patch = RuntimeCaller::<EHF>::new(&code[..]).get_named_preset(Some(name))?;
+				let patch = RuntimeCaller::<EHF>::new(&code[..], Default::default())
+					.get_named_preset(Some(name))?;
 				Ok(Genesis::RuntimeGenesis(RuntimeGenesisInner {
 					json_blob: RuntimeGenesisConfigJson::Patch(patch),
 					code: code.clone(),
@@ -158,7 +159,7 @@ where
 				json_blob: RuntimeGenesisConfigJson::Config(config),
 				code,
 			}) => {
-				RuntimeCaller::<EHF>::new(&code[..])
+				RuntimeCaller::<EHF>::new(&code[..], self.executor_params.clone())
 					.get_storage_for_config(config)?
 					.assimilate_storage(storage)?;
 				storage
@@ -169,7 +170,7 @@ where
 				json_blob: RuntimeGenesisConfigJson::Patch(patch),
 				code,
 			}) => {
-				RuntimeCaller::<EHF>::new(&code[..])
+				RuntimeCaller::<EHF>::new(&code[..], self.executor_params.clone())
 					.get_storage_for_patch(patch)?
 					.assimilate_storage(storage)?;
 				storage
@@ -314,6 +315,7 @@ pub struct ChainSpecBuilder<E = NoExtension, EHF = ()> {
 	protocol_id: Option<String>,
 	fork_id: Option<String>,
 	properties: Option<Properties>,
+	heap_pages: Option<u64>,
 }
 
 impl<E, EHF> ChainSpecBuilder<E, EHF> {
@@ -331,6 +333,7 @@ impl<E, EHF> ChainSpecBuilder<E, EHF> {
 			protocol_id: None,
 			fork_id: None,
 			properties: None,
+			heap_pages: None,
 		}
 	}
 
@@ -413,6 +416,12 @@ impl<E, EHF> ChainSpecBuilder<E, EHF> {
 		self
 	}
 
+	/// Sets the number of heap pages available to the executor during chain spec building.
+	pub fn with_heap_pages(mut self, pages: u64) -> Self {
+		self.heap_pages = Some(pages);
+		self
+	}
+
 	/// Builds a [`ChainSpec`] instance using the provided settings.
 	pub fn build(self) -> ChainSpec<E, EHF> {
 		let client_spec = ClientSpec {
@@ -430,9 +439,12 @@ impl<E, EHF> ChainSpecBuilder<E, EHF> {
 			code_substitutes: BTreeMap::new(),
 		};
 
+		let executor_params = OffchainExecutorParams { extra_heap_pages: self.heap_pages };
+
 		ChainSpec {
 			client_spec,
 			genesis: GenesisSource::GenesisBuilderApi(self.genesis_build_action, self.code.into()),
+			executor_params,
 			_host_functions: Default::default(),
 		}
 	}
@@ -446,6 +458,7 @@ impl<E, EHF> ChainSpecBuilder<E, EHF> {
 pub struct ChainSpec<E = NoExtension, EHF = ()> {
 	client_spec: ClientSpec<E>,
 	genesis: GenesisSource<EHF>,
+	executor_params: OffchainExecutorParams,
 	_host_functions: PhantomData<EHF>,
 }
 
@@ -454,6 +467,7 @@ impl<E: Clone, EHF> Clone for ChainSpec<E, EHF> {
 		ChainSpec {
 			client_spec: self.client_spec.clone(),
 			genesis: self.genesis.clone(),
+			executor_params: self.executor_params.clone(),
 			_host_functions: self._host_functions,
 		}
 	}
@@ -533,6 +547,7 @@ impl<E: serde::de::DeserializeOwned, EHF> ChainSpec<E, EHF> {
 		Ok(ChainSpec {
 			client_spec,
 			genesis: GenesisSource::Binary(json),
+			executor_params: Default::default(),
 			_host_functions: Default::default(),
 		})
 	}
@@ -556,6 +571,7 @@ impl<E: serde::de::DeserializeOwned, EHF> ChainSpec<E, EHF> {
 		Ok(ChainSpec {
 			client_spec,
 			genesis: GenesisSource::File(path),
+			executor_params: Default::default(),
 			_host_functions: Default::default(),
 		})
 	}
@@ -586,7 +602,8 @@ where
 				}),
 			) => {
 				let mut storage =
-					RuntimeCaller::<EHF>::new(&code[..]).get_storage_for_config(config)?;
+					RuntimeCaller::<EHF>::new(&code[..], self.executor_params.clone())
+						.get_storage_for_config(config)?;
 				storage.top.insert(sp_core::storage::well_known_keys::CODE.to_vec(), code);
 				RawGenesis::from(storage)
 			},
@@ -598,7 +615,8 @@ where
 				}),
 			) => {
 				let mut storage =
-					RuntimeCaller::<EHF>::new(&code[..]).get_storage_for_patch(patch)?;
+					RuntimeCaller::<EHF>::new(&code[..], self.executor_params.clone())
+						.get_storage_for_patch(patch)?;
 				storage.top.insert(sp_core::storage::well_known_keys::CODE.to_vec(), code);
 				RawGenesis::from(storage)
 			},
@@ -692,6 +710,10 @@ where
 			.map(|(h, c)| (h.clone(), c.0.clone()))
 			.collect()
 	}
+
+	fn set_executor_params(&mut self, executor_params: OffchainExecutorParams) {
+		self.executor_params = executor_params;
+	}
 }
 
 /// The `fun` will be called with the value at `path`.

substrate/client/chain-spec/src/genesis_config_builder.rs

@@ -31,6 +31,8 @@ pub use sp_genesis_builder::{DEV_RUNTIME_PRESET, LOCAL_TESTNET_RUNTIME_PRESET};
 use sp_state_machine::BasicExternalities;
 use std::borrow::Cow;
 
+use crate::OffchainExecutorParams;
+
 /// A utility that facilitates calling the GenesisBuilder API from the runtime wasm code blob.
 ///
 /// `EHF` type allows to specify the extended host function required for building runtime's genesis
@@ -60,13 +62,19 @@ where
 	/// Creates new instance using the provided code blob.
 	///
 	/// This code is later referred to as `runtime`.
-	pub fn new(code: &'a [u8]) -> Self {
+	pub fn new(code: &'a [u8], executor_params: OffchainExecutorParams) -> Self {
+		let mut executor = WasmExecutor::<(sp_io::SubstrateHostFunctions, EHF)>::builder()
+			.with_allow_missing_host_functions(true);
+		if let Some(heap_pages) = executor_params.extra_heap_pages {
+			executor =
+				executor.with_offchain_heap_alloc_strategy(sc_executor::HeapAllocStrategy::Static {
+					extra_pages: heap_pages as _,
+				})
+		};
 		GenesisConfigBuilderRuntimeCaller {
 			code: code.into(),
 			code_hash: sp_crypto_hashing::blake2_256(code).to_vec(),
-			executor: WasmExecutor::<(sp_io::SubstrateHostFunctions, EHF)>::builder()
-				.with_allow_missing_host_functions(true)
-				.build(),
+			executor: executor.build(),
 		}
 	}
 
@@ -183,30 +191,36 @@ mod tests {
 	#[test]
 	fn list_presets_works() {
 		sp_tracing::try_init_simple();
-		let presets =
-			<GenesisConfigBuilderRuntimeCaller>::new(substrate_test_runtime::wasm_binary_unwrap())
-				.preset_names()
-				.unwrap();
+		let presets = <GenesisConfigBuilderRuntimeCaller>::new(
+			substrate_test_runtime::wasm_binary_unwrap(),
+			Default::default(),
+		)
+		.preset_names()
+		.unwrap();
 		assert_eq!(presets, vec![PresetId::from("foobar"), PresetId::from("staging"),]);
 	}
 
 	#[test]
 	fn get_default_config_works() {
-		let config =
-			<GenesisConfigBuilderRuntimeCaller>::new(substrate_test_runtime::wasm_binary_unwrap())
-				.get_default_config()
-				.unwrap();
+		let config = <GenesisConfigBuilderRuntimeCaller>::new(
+			substrate_test_runtime::wasm_binary_unwrap(),
+			Default::default(),
+		)
+		.get_default_config()
+		.unwrap();
 		let expected = r#"{"babe": {"authorities": [], "epochConfig": {"allowed_slots": "PrimaryAndSecondaryVRFSlots", "c": [1, 4]}}, "balances": {"balances": []}, "substrateTest": {"authorities": []}, "system": {}}"#;
 		assert_eq!(from_str::<Value>(expected).unwrap(), config);
 	}
 
 	#[test]
 	fn get_named_preset_works() {
 		sp_tracing::try_init_simple();
-		let config =
-			<GenesisConfigBuilderRuntimeCaller>::new(substrate_test_runtime::wasm_binary_unwrap())
-				.get_named_preset(Some(&"foobar".to_string()))
-				.unwrap();
+		let config = <GenesisConfigBuilderRuntimeCaller>::new(
+			substrate_test_runtime::wasm_binary_unwrap(),
+			Default::default(),
+		)
+		.get_named_preset(Some(&"foobar".to_string()))
+		.unwrap();
 		let expected = r#"{"foo":"bar"}"#;
 		assert_eq!(from_str::<Value>(expected).unwrap(), config);
 	}
@@ -225,10 +239,12 @@ mod tests {
 			},
 		});
 
-		let storage =
-			<GenesisConfigBuilderRuntimeCaller>::new(substrate_test_runtime::wasm_binary_unwrap())
-				.get_storage_for_patch(patch)
-				.unwrap();
+		let storage = <GenesisConfigBuilderRuntimeCaller>::new(
+			substrate_test_runtime::wasm_binary_unwrap(),
+			Default::default(),
+		)
+		.get_storage_for_patch(patch)
+		.unwrap();
 
 		//Babe|Authorities
 		let value: Vec<u8> = storage

substrate/client/chain-spec/src/lib.rs

@@ -377,6 +377,14 @@ pub enum ChainType {
 	Custom(String),
 }
 
+/// Executor parameters used to call into the runtime when building a chain spec.
+#[derive(Default, Clone)]
+pub struct OffchainExecutorParams {
+	/// Number of extra heap pages available to the executor. If not specified, executor's
+	/// default is used.
+	pub extra_heap_pages: Option<u64>,
+}
+
 impl Default for ChainType {
 	fn default() -> Self {
 		Self::Live
@@ -424,6 +432,8 @@ pub trait ChainSpec: BuildStorage + Send + Sync {
 	fn set_storage(&mut self, storage: Storage);
 	/// Returns code substitutes that should be used for the on chain wasm.
 	fn code_substitutes(&self) -> std::collections::BTreeMap<String, Vec<u8>>;
+	/// Sets executor parameters that should be used when building this chain spec.
+	fn set_executor_params(&mut self, executor_params: OffchainExecutorParams);
 }
 
 impl std::fmt::Debug for dyn ChainSpec {

substrate/client/cli/Cargo.toml

@@ -34,6 +34,7 @@ serde_json = { workspace = true, default-features = true }
 thiserror = { workspace = true }
 # personal fork here as workaround for: https://github.com/rust-bitcoin/rust-bip39/pull/64
 bip39 = { package = "parity-bip39", version = "2.0.1", features = ["rand"] }
+sc-chain-spec = { workspace = true, default-features = true }
 sc-client-api = { workspace = true, default-features = true }
 sc-client-db = { workspace = true }
 sc-keystore = { workspace = true, default-features = true }

substrate/client/cli/src/commands/build_spec_cmd.rs

@@ -23,6 +23,7 @@ use crate::{
 };
 use clap::Parser;
 use log::info;
+use sc_chain_spec::OffchainExecutorParams;
 use sc_network::config::build_multiaddr;
 use sc_service::{
 	config::{MultiaddrWithPeerId, NetworkConfiguration},
@@ -43,6 +44,10 @@ pub struct BuildSpecCmd {
 	#[arg(long)]
 	pub disable_default_bootnode: bool,
 
+	/// Number of extra heap pages available to genesis builder.
+	#[arg(long)]
+	pub extra_heap_pages: Option<u64>,
+
 	#[allow(missing_docs)]
 	#[clap(flatten)]
 	pub shared_params: SharedParams,
@@ -72,6 +77,8 @@ impl BuildSpecCmd {
 			spec.add_boot_node(addr)
 		}
 
+		spec.set_executor_params(OffchainExecutorParams { extra_heap_pages: self.extra_heap_pages });
+
 		let json = sc_service::chain_ops::build_spec(&*spec, raw_output)?;
 		if std::io::stdout().write_all(json.as_bytes()).is_err() {
 			let _ = std::io::stderr().write_all(b"Error writing to stdout\n");

substrate/utils/frame/benchmarking-cli/src/shared/genesis_state.rs

@@ -116,7 +116,7 @@ fn genesis_from_code<EHF: HostFunctions>(
 		sp_io::SubstrateHostFunctions,
 		frame_benchmarking::benchmarking::HostFunctions,
 		EHF,
-	)>::new(code);
+	)>::new(code, Default::default());
 
 	let mut preset_json = genesis_config_caller.get_named_preset(Some(genesis_builder_preset))?;
 	if let Some(patcher) = storage_patcher {