diff --git a/polkadot/parachain/src/lib.rs b/polkadot/parachain/src/lib.rs index 913d887e4a8a..bd75296bf837 100644 --- a/polkadot/parachain/src/lib.rs +++ b/polkadot/parachain/src/lib.rs @@ -14,8 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -#![warn(unused_crate_dependencies)] - //! Defines primitive types for creating or validating a parachain. //! //! When compiled with standard library support, this crate exports a `wasm` diff --git a/substrate/utils/wasm-builder/README.md b/substrate/utils/wasm-builder/README.md index db32f5cbc955..6dfc743816cb 100644 --- a/substrate/utils/wasm-builder/README.md +++ b/substrate/utils/wasm-builder/README.md @@ -13,7 +13,7 @@ A project that should be compiled as a Wasm binary needs to: The `build.rs` file needs to contain the following code: -```rust +```rust,no_run fn main() { #[cfg(feature = "std")] { @@ -32,7 +32,7 @@ fn main() { As the final step, you need to add the following to your project: -```rust +```rust,ignore include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); ``` @@ -63,11 +63,17 @@ By using environment variables, you can configure which Wasm binaries are built - `WASM_TARGET_DIRECTORY` - Will copy any build Wasm binary to the given directory. The path needs to be absolute. - `WASM_BUILD_TOOLCHAIN` - The toolchain that should be used to build the Wasm binaries. The format needs to be the same as used by cargo, e.g. `nightly-2020-02-20`. +- `WASM_BUILD_WORKSPACE_HINT` - Hint the workspace that is being built. This is normally not required as we walk up from + the target directory until we find a `Cargo.toml`. If the target directory is changed for + the build, this environment variable can be used to point to the actual workspace. +- `WASM_BUILD_STD` - Sets whether the Rust's standard library crates will also be built. This is necessary to make sure + the standard library crates only use the exact WASM feature set that our executor supports. + Enabled by default. - `CARGO_NET_OFFLINE` - If `true`, `--offline` will be passed to all processes launched to prevent network access. Useful in offline environments. Each project can be skipped individually by using the environment variable `SKIP_PROJECT_NAME_WASM_BUILD`. Where -`PROJECT_NAME` needs to be replaced by the name of the cargo project, e.g. `node-runtime` will be `NODE_RUNTIME`. +`PROJECT_NAME` needs to be replaced by the name of the cargo project, e.g. `kitchensink-runtime` will be `NODE_RUNTIME`. ## Prerequisites diff --git a/substrate/utils/wasm-builder/src/lib.rs b/substrate/utils/wasm-builder/src/lib.rs index c9011f97be71..cd31a8cba105 100644 --- a/substrate/utils/wasm-builder/src/lib.rs +++ b/substrate/utils/wasm-builder/src/lib.rs @@ -15,99 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! # Wasm builder is a utility for building a project as a Wasm binary -//! -//! The Wasm builder is a tool that integrates the process of building the WASM binary of your -//! project into the main `cargo` build process. -//! -//! ## Project setup -//! -//! A project that should be compiled as a Wasm binary needs to: -//! -//! 1. Add a `build.rs` file. -//! 2. Add `wasm-builder` as dependency into `build-dependencies`. -//! -//! The `build.rs` file needs to contain the following code: -//! -//! ```no_run -//! use substrate_wasm_builder::WasmBuilder; -//! -//! fn main() { -//! WasmBuilder::new() -//! // Tell the builder to build the project (crate) this `build.rs` is part of. -//! .with_current_project() -//! // Make sure to export the `heap_base` global, this is required by Substrate -//! .export_heap_base() -//! // Build the Wasm file so that it imports the memory (need to be provided by at instantiation) -//! .import_memory() -//! // Build it. -//! .build() -//! } -//! ``` -//! -//! As the final step, you need to add the following to your project: -//! -//! ```ignore -//! include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -//! ``` -//! -//! This will include the generated Wasm binary as two constants `WASM_BINARY` and -//! `WASM_BINARY_BLOATY`. The former is a compact Wasm binary and the latter is the Wasm binary as -//! being generated by the compiler. Both variables have `Option<&'static [u8]>` as type. -//! -//! ### Feature -//! -//! Wasm builder supports to enable cargo features while building the Wasm binary. By default it -//! will enable all features in the wasm build that are enabled for the native build except the -//! `default` and `std` features. Besides that, wasm builder supports the special `runtime-wasm` -//! feature. This `runtime-wasm` feature will be enabled by the wasm builder when it compiles the -//! Wasm binary. If this feature is not present, it will not be enabled. -//! -//! ## Environment variables -//! -//! By using environment variables, you can configure which Wasm binaries are built and how: -//! -//! - `SKIP_WASM_BUILD` - Skips building any Wasm binary. This is useful when only native should be -//! recompiled. If this is the first run and there doesn't exist a Wasm binary, this will set both -//! variables to `None`. -//! - `WASM_BUILD_TYPE` - Sets the build type for building Wasm binaries. Supported values are -//! `release` or `debug`. By default the build type is equal to the build type used by the main -//! build. -//! - `FORCE_WASM_BUILD` - Can be set to force a Wasm build. On subsequent calls the value of the -//! variable needs to change. As wasm-builder instructs `cargo` to watch for file changes this -//! environment variable should only be required in certain circumstances. -//! - `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm -//! binary. -//! - `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build. -//! - `WASM_TARGET_DIRECTORY` - Will copy any build Wasm binary to the given directory. The path -//! needs to be absolute. -//! - `WASM_BUILD_TOOLCHAIN` - The toolchain that should be used to build the Wasm binaries. The -//! format needs to be the same as used by cargo, e.g. `nightly-2020-02-20`. -//! - `WASM_BUILD_WORKSPACE_HINT` - Hint the workspace that is being built. This is normally not -//! required as we walk up from the target directory until we find a `Cargo.toml`. If the target -//! directory is changed for the build, this environment variable can be used to point to the -//! actual workspace. -//! - `CARGO_NET_OFFLINE` - If `true`, `--offline` will be passed to all processes launched to -//! prevent network access. Useful in offline environments. -//! -//! Each project can be skipped individually by using the environment variable -//! `SKIP_PROJECT_NAME_WASM_BUILD`. Where `PROJECT_NAME` needs to be replaced by the name of the -//! cargo project, e.g. `kitchensink-runtime` will be `NODE_RUNTIME`. -//! -//! ## Prerequisites: -//! -//! Wasm builder requires the following prerequisites for building the Wasm binary: -//! -//! - rust nightly + `wasm32-unknown-unknown` toolchain -//! -//! or -//! -//! - rust stable and version at least 1.68.0 + `wasm32-unknown-unknown` toolchain -//! -//! If a specific rust is installed with `rustup`, it is important that the wasm target is -//! installed as well. For example if installing the rust from 20.02.2020 using `rustup -//! install nightly-2020-02-20`, the wasm target needs to be installed as well `rustup target add -//! wasm32-unknown-unknown --toolchain nightly-2020-02-20`. +#![doc = include_str!("../README.md")] use std::{ env, fs, @@ -158,6 +66,9 @@ const FORCE_WASM_BUILD_ENV: &str = "FORCE_WASM_BUILD"; /// Environment variable that hints the workspace we are building. const WASM_BUILD_WORKSPACE_HINT: &str = "WASM_BUILD_WORKSPACE_HINT"; +/// Environment variable to set whether we'll build `core`/`std`. +const WASM_BUILD_STD: &str = "WASM_BUILD_STD"; + /// Write to the given `file` if the `content` is different. fn write_file_if_changed(file: impl AsRef, content: impl AsRef) { if fs::read_to_string(file.as_ref()).ok().as_deref() != Some(content.as_ref()) { @@ -282,6 +193,12 @@ impl CargoCommand { self.version } + /// Returns whether this version of the toolchain supports nightly features. + fn supports_nightly_features(&self) -> bool { + self.version.map_or(false, |version| version.is_nightly) || + env::var("RUSTC_BOOTSTRAP").is_ok() + } + /// Check if the supplied cargo command supports our Substrate wasm environment. /// /// This means that either the cargo version is at minimum 1.68.0 or this is a nightly cargo. @@ -332,3 +249,26 @@ impl std::ops::Deref for CargoCommandVersioned { fn color_output_enabled() -> bool { env::var(crate::WASM_BUILD_NO_COLOR).is_err() } + +/// Fetches a boolean environment variable. Will exit the process if the value is invalid. +fn get_bool_environment_variable(name: &str) -> Option { + let value = env::var_os(name)?; + + // We're comparing `OsString`s here so we can't use a `match`. + if value == "1" { + Some(true) + } else if value == "0" { + Some(false) + } else { + build_helper::warning!( + "the '{}' environment variable has an invalid value; it must be either '1' or '0'", + name + ); + std::process::exit(1); + } +} + +/// Returns whether we need to also compile the standard library when compiling the runtime. +fn build_std_required() -> bool { + crate::get_bool_environment_variable(crate::WASM_BUILD_STD).unwrap_or(true) +} diff --git a/substrate/utils/wasm-builder/src/prerequisites.rs b/substrate/utils/wasm-builder/src/prerequisites.rs index 8e81e6774faa..2cdbdd2798eb 100644 --- a/substrate/utils/wasm-builder/src/prerequisites.rs +++ b/substrate/utils/wasm-builder/src/prerequisites.rs @@ -47,14 +47,11 @@ pub(crate) fn check() -> Result { check_wasm_toolchain_installed(cargo_command) } -/// Create the project that will be used to check that the wasm toolchain is installed and to -/// extract the rustc version. -fn create_check_toolchain_project(project_dir: &Path) { - let lib_rs_file = project_dir.join("src/lib.rs"); - let main_rs_file = project_dir.join("src/main.rs"); - let build_rs_file = project_dir.join("build.rs"); - let manifest_path = project_dir.join("Cargo.toml"); +/// Creates a minimal dummy crate at the given path and returns the manifest path. +fn create_minimal_crate(project_dir: &Path) -> std::path::PathBuf { + fs::create_dir_all(project_dir.join("src")).expect("Creating src dir does not fail; qed"); + let manifest_path = project_dir.join("Cargo.toml"); write_file_if_changed( &manifest_path, r#" @@ -62,120 +59,99 @@ fn create_check_toolchain_project(project_dir: &Path) { name = "wasm-test" version = "1.0.0" edition = "2021" - build = "build.rs" - - [lib] - name = "wasm_test" - crate-type = ["cdylib"] [workspace] "#, ); - write_file_if_changed(lib_rs_file, "pub fn test() {}"); - - // We want to know the rustc version of the rustc that is being used by our cargo command. - // The cargo command is determined by some *very* complex algorithm to find the cargo command - // that supports nightly. - // The best solution would be if there is a `cargo rustc --version` command, which sadly - // doesn't exists. So, the only available way of getting the rustc version is to build a project - // and capture the rustc version in this build process. This `build.rs` is exactly doing this. - // It gets the rustc version by calling `rustc --version` and exposing it in the `RUSTC_VERSION` - // environment variable. - write_file_if_changed( - build_rs_file, - r#" - fn main() { - let rustc_cmd = std::env::var("RUSTC").ok().unwrap_or_else(|| "rustc".into()); - - let rustc_version = std::process::Command::new(rustc_cmd) - .arg("--version") - .output() - .ok() - .and_then(|o| String::from_utf8(o.stdout).ok()); - - println!( - "cargo:rustc-env=RUSTC_VERSION={}", - rustc_version.unwrap_or_else(|| "unknown rustc version".into()), - ); - } - "#, - ); - // Just prints the `RURSTC_VERSION` environment variable that is being created by the - // `build.rs` script. - write_file_if_changed( - main_rs_file, - r#" - fn main() { - println!("{}", env!("RUSTC_VERSION")); - } - "#, - ); + + write_file_if_changed(project_dir.join("src/main.rs"), "fn main() {}"); + manifest_path } fn check_wasm_toolchain_installed( cargo_command: CargoCommand, ) -> Result { let temp = tempdir().expect("Creating temp dir does not fail; qed"); - fs::create_dir_all(temp.path().join("src")).expect("Creating src dir does not fail; qed"); - create_check_toolchain_project(temp.path()); - - let err_msg = print_error_message("Rust WASM toolchain not installed, please install it!"); - let manifest_path = temp.path().join("Cargo.toml").display().to_string(); - - let mut build_cmd = cargo_command.command(); - // Chdir to temp to avoid including project's .cargo/config.toml - // by accident - it can happen in some CI environments. - build_cmd.current_dir(&temp); - build_cmd.args(&[ - "build", - "--target=wasm32-unknown-unknown", - "--manifest-path", - &manifest_path, - ]); + let manifest_path = create_minimal_crate(temp.path()).display().to_string(); + + let prepare_command = |subcommand| { + let mut cmd = cargo_command.command(); + // Chdir to temp to avoid including project's .cargo/config.toml + // by accident - it can happen in some CI environments. + cmd.current_dir(&temp); + cmd.args(&[ + subcommand, + "--target=wasm32-unknown-unknown", + "--manifest-path", + &manifest_path, + ]); + + if super::color_output_enabled() { + cmd.arg("--color=always"); + } - if super::color_output_enabled() { - build_cmd.arg("--color=always"); + // manually set the `CARGO_TARGET_DIR` to prevent a cargo deadlock + let target_dir = temp.path().join("target").display().to_string(); + cmd.env("CARGO_TARGET_DIR", &target_dir); + + // Make sure the host's flags aren't used here, e.g. if an alternative linker is specified + // in the RUSTFLAGS then the check we do here will break unless we clear these. + cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); + cmd.env_remove("RUSTFLAGS"); + cmd + }; + + let err_msg = + print_error_message("Rust WASM toolchain is not properly installed; please install it!"); + let build_result = prepare_command("build").output().map_err(|_| err_msg.clone())?; + if !build_result.status.success() { + return match String::from_utf8(build_result.stderr) { + Ok(ref err) if err.contains("the `wasm32-unknown-unknown` target may not be installed") => + Err(print_error_message("Cannot compile the WASM runtime: the `wasm32-unknown-unknown` target is not installed!\n\ + You can install it with `rustup target add wasm32-unknown-unknown` if you're using `rustup`.")), + + // Apparently this can happen when we're running on a non Tier 1 platform. + Ok(ref err) if err.contains("linker `rust-lld` not found") => + Err(print_error_message("Cannot compile the WASM runtime: `rust-lld` not found!")), + + Ok(ref err) => Err(format!( + "{}\n\n{}\n{}\n{}{}\n", + err_msg, + Color::Yellow.bold().paint("Further error information:"), + Color::Yellow.bold().paint("-".repeat(60)), + err, + Color::Yellow.bold().paint("-".repeat(60)), + )), + + Err(_) => Err(err_msg), + }; } - let mut run_cmd = cargo_command.command(); - // Chdir to temp to avoid including project's .cargo/config.toml - // by accident - it can happen in some CI environments. - run_cmd.current_dir(&temp); - run_cmd.args(&["run", "--manifest-path", &manifest_path]); - - // manually set the `CARGO_TARGET_DIR` to prevent a cargo deadlock - let target_dir = temp.path().join("target").display().to_string(); - build_cmd.env("CARGO_TARGET_DIR", &target_dir); - run_cmd.env("CARGO_TARGET_DIR", &target_dir); - - // Make sure the host's flags aren't used here, e.g. if an alternative linker is specified - // in the RUSTFLAGS then the check we do here will break unless we clear these. - build_cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); - run_cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); - build_cmd.env_remove("RUSTFLAGS"); - run_cmd.env_remove("RUSTFLAGS"); - - build_cmd.output().map_err(|_| err_msg.clone()).and_then(|s| { - if s.status.success() { - let version = run_cmd.output().ok().and_then(|o| String::from_utf8(o.stdout).ok()); - Ok(CargoCommandVersioned::new( - cargo_command, - version.unwrap_or_else(|| "unknown rustc version".into()), - )) - } else { - match String::from_utf8(s.stderr) { - Ok(ref err) if err.contains("linker `rust-lld` not found") => - Err(print_error_message("`rust-lld` not found, please install it!")), - Ok(ref err) => Err(format!( - "{}\n\n{}\n{}\n{}{}\n", - err_msg, - Color::Yellow.bold().paint("Further error information:"), - Color::Yellow.bold().paint("-".repeat(60)), - err, - Color::Yellow.bold().paint("-".repeat(60)), - )), - Err(_) => Err(err_msg), + let mut run_cmd = prepare_command("rustc"); + run_cmd.args(&["-q", "--", "--version"]); + + let version = run_cmd + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .unwrap_or_else(|| "unknown rustc version".into()); + + if crate::build_std_required() { + let mut sysroot_cmd = prepare_command("rustc"); + sysroot_cmd.args(&["-q", "--", "--print", "sysroot"]); + if let Some(sysroot) = + sysroot_cmd.output().ok().and_then(|o| String::from_utf8(o.stdout).ok()) + { + let src_path = + Path::new(sysroot.trim()).join("lib").join("rustlib").join("src").join("rust"); + if !src_path.exists() { + return Err(print_error_message( + "Cannot compile the WASM runtime: no standard library sources found!\n\ + You can install them with `rustup component add rust-src` if you're using `rustup`.", + )) } } - }) + } + + Ok(CargoCommandVersioned::new(cargo_command, version)) } diff --git a/substrate/utils/wasm-builder/src/version.rs b/substrate/utils/wasm-builder/src/version.rs index e4f7d98be618..3a0a306d737d 100644 --- a/substrate/utils/wasm-builder/src/version.rs +++ b/substrate/utils/wasm-builder/src/version.rs @@ -212,4 +212,21 @@ mod tests { version_1_69_0, ); } + + #[test] + fn parse_rustc_version() { + let version = Version::extract("rustc 1.73.0 (cc66ad468 2023-10-03)").unwrap(); + assert_eq!( + version, + Version { + major: 1, + minor: 73, + patch: 0, + is_nightly: false, + year: Some(2023), + month: Some(10), + day: Some(03), + } + ); + } } diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 2e6f671c45ed..5bf44c2c9b20 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -750,6 +750,25 @@ fn build_bloaty_blob( build_cmd.arg("--offline"); } + // Our executor currently only supports the WASM MVP feature set, however nowadays + // when compiling WASM the Rust compiler has more features enabled by default. + // + // We do set the `-C target-cpu=mvp` flag to make sure that *our* code gets compiled + // in a way that is compatible with our executor, however this doesn't affect Rust's + // standard library crates (`std`, `core` and `alloc`) which are by default precompiled + // and still can make use of these extra features. + // + // So here we force the compiler to also compile the standard library crates for us + // to make sure that they also only use the MVP features. + if crate::build_std_required() { + // Unfortunately this is still a nightly-only flag, but FWIW it is pretty widely used + // so it's unlikely to break without a replacement. + build_cmd.arg("-Z").arg("build-std"); + if !cargo_cmd.supports_nightly_features() { + build_cmd.env("RUSTC_BOOTSTRAP", "1"); + } + } + println!("{}", colorize_info_message("Information that should be included in a bug report.")); println!("{} {:?}", colorize_info_message("Executing build command:"), build_cmd); println!("{} {}", colorize_info_message("Using rustc version:"), cargo_cmd.rustc_version()); @@ -952,6 +971,7 @@ fn generate_rerun_if_changed_instructions( println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_RUSTFLAGS_ENV); println!("cargo:rerun-if-env-changed={}", crate::WASM_TARGET_DIRECTORY); println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_TOOLCHAIN); + println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_STD); } /// Track files and paths related to the given package to rerun `build.rs` on any relevant change.