Skip to content

Commit

Permalink
Compile for wasm32-unknown instead of wasm32-wasi (#1129)
Browse files Browse the repository at this point in the history 
* Compile for `wasm32-unknown` instead of `wasm32-wasi`

* No longer use rustup in prepare.mjs

* Update comments

* Install the wasm32-unknown target on CI
  • Loading branch information
@tomaka
tomaka authored Sep 9, 2023
1 parent 758188e commit d353b6c
Show file tree
Hide file tree
Showing 10 changed files with 83 additions and 254 deletions.
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ jobs:
image: rust:1.72
steps:
- uses: actions/checkout@v4
- run: rustup target add wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- uses: actions/[email protected]
with:
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/deploy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ jobs:
- uses: actions/checkout@v4
with:
path: repo
- run: rustup target add wasm32-unknown-unknown
- uses: actions/[email protected]
with:
node-version: current
Expand Down Expand Up @@ -176,6 +177,7 @@ jobs:
image: rust:1.72
steps:
- uses: actions/checkout@v4
- run: rustup target add wasm32-unknown-unknown
- uses: actions/[email protected]
with:
# Set the oldest version still maintained, in order to ensure compatibility. See <https://nodejs.dev/en/about/releases/>
Expand Down Expand Up @@ -204,6 +206,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Necessary below for checking if the tag exists.
- run: rustup target add wasm32-unknown-unknown
- uses: actions/[email protected]
with:
node-version: 16
Expand Down
1 change: 0 additions & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

37 changes: 6 additions & 31 deletions wasm-node/javascript/prepare.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,34 +34,8 @@ if (process.argv.slice(2).indexOf("--release") !== -1) {
if (buildProfile != 'debug' && buildProfile != 'min-size-release')
throw new Error("Either --debug or --release must be passed");

// The Rust version to use.
// The Rust version is pinned because the wasi target is still unstable. Without pinning, it is
// possible for the wasm-js bindings to change between two Rust versions. Feel free to update
// this version pin whenever you like, provided it continues to build.
const rustVersion = '1.72.0';

// Assume that the user has `rustup` installed and make sure that `rust_version` is available.
// Because `rustup install` requires an Internet connection, check whether the toolchain is
// already installed before attempting it.
try {
child_process.execSync(
"rustup which --toolchain " + rustVersion + " cargo",
{ 'stdio': 'inherit' }
);
} catch (error) {
child_process.execSync(
"rustup install --no-self-update --profile=minimal " + rustVersion,
{ 'stdio': 'inherit' }
);
}
// `rustup target add` doesn't require an Internet connection if the target is already installed.
child_process.execSync(
"rustup target add --toolchain=" + rustVersion + " wasm32-wasi",
{ 'stdio': 'inherit' }
);

// The important step in this script is running `cargo build --target wasm32-wasi` on the Rust
// code. This generates a `wasm` file in `target/wasm32-wasi`.
// The important step in this script is running `cargo build --target wasm32-unknown-unknown` on
// the Rust code. This generates a `wasm` file in `target/wasm32-unknown-unknown`.
// Some optional Wasm features are enabled during the compilation in order to speed up the
// execution of smoldot.
// SIMD is intentionally not enabled, because WASM engines seem to allow only SIMD instructions
Expand All @@ -72,16 +46,17 @@ child_process.execSync(
// precompiled), but the missing optimizations shouldn't be too much of a problem. The Rust
// standard library could be compiled with these features using the `-Z build-std` flag, but at
// the time of the writing of this comment this would require an unstable version of Rust.
// Use `rustc --print target-features --target wasm32-wasi` to see the list of target features.
// Use `rustc --print target-features --target wasm32-unknown-unknown` to see the list of target
// features.
// See <https://webassembly.org/roadmap/> to know which version of which engine supports which
// feature.
// See also the issue: <https://github.com/smol-dot/smoldot/issues/350>
child_process.execSync(
"cargo +" + rustVersion + " build --package smoldot-light-wasm --target wasm32-wasi --no-default-features " +
"cargo build --package smoldot-light-wasm --target wasm32-unknown-unknown --no-default-features " +
(buildProfile == 'debug' ? '' : ("--profile " + buildProfile)),
{ 'stdio': 'inherit', 'env': { 'RUSTFLAGS': '-C target-feature=+bulk-memory,+sign-ext', ...process.env } }
);
const rustOutput = "../../target/wasm32-wasi/" + buildProfile + "/smoldot_light_wasm.wasm";
const rustOutput = "../../target/wasm32-unknown-unknown/" + buildProfile + "/smoldot_light_wasm.wasm";

// The code below will write a variable number of files to the `src/internals/bytecode` directory.
// Start by clearing all existing files from this directory in case there are some left from past
Expand Down
211 changes: 28 additions & 183 deletions wasm-node/javascript/src/internals/local-instance.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,34 @@ export async function startLocalInstance(config: Config, wasmModule: WebAssembly
throw new Error();
},

random_get: (ptr: number, len: number) => {
const instance = state.instance!;

ptr >>>= 0;
len >>>= 0;

const baseBuffer = new Uint8Array(instance.exports.memory.buffer)
.subarray(ptr, ptr + len);
for (let iter = 0; iter < len; iter += 65536) {
// `baseBuffer.subarray` automatically saturates at the end of the buffer
config.getRandomValues(baseBuffer.subarray(iter, iter + 65536))
}
},

unix_timestamp_us: (): bigint => {
const value = Math.floor(Date.now());
if (value < 0) throw new Error("UNIX timestamp inferior to 0");
return BigInt(value) * BigInt(1_000);
},

monotonic_clock_us: (): bigint => {
const nowMs = config.performanceNow();
const nowMsInt = Math.floor(nowMs);
const now = BigInt(nowMsInt) * BigInt(1_000) +
BigInt(Math.floor(((nowMs - nowMsInt) * 1_000)));
return now;
},

buffer_size: (bufferIndex: number) => {
const buf = state.bufferIndices[bufferIndex]!;
return buf.byteLength;
Expand Down Expand Up @@ -358,195 +386,12 @@ export async function startLocalInstance(config: Config, wasmModule: WebAssembly
}
};

const wasiBindings = {
// Need to fill the buffer described by `ptr` and `len` with random data.
// This data will be used in order to generate secrets. Do not use a dummy implementation!
random_get: (ptr: number, len: number) => {
const instance = state.instance!;

ptr >>>= 0;
len >>>= 0;

const baseBuffer = new Uint8Array(instance.exports.memory.buffer)
.subarray(ptr, ptr + len);
for (let iter = 0; iter < len; iter += 65536) {
// `baseBuffer.subarray` automatically saturates at the end of the buffer
config.getRandomValues(baseBuffer.subarray(iter, iter + 65536))
}

return 0;
},

clock_time_get: (clockId: number, _precision: bigint, outPtr: number): number => {
// See <https://github.com/rust-lang/rust/blob/master/library/std/src/sys/wasi/time.rs>
// and <docs.rs/wasi/> for help.

const instance = state.instance!;
const mem = new Uint8Array(instance.exports.memory.buffer);
outPtr >>>= 0;

// We ignore the precision, as it can't be implemented anyway.

switch (clockId) {
case 0: {
// Realtime clock.
const now = BigInt(Math.floor(Date.now())) * BigInt(1_000_000);
buffer.writeUInt64LE(mem, outPtr, now)

// Success.
return 0;
}
case 1: {
// Monotonic clock.
const nowMs = config.performanceNow();
const nowMsInt = Math.floor(nowMs);
const now = BigInt(nowMsInt) * BigInt(1_000_000) +
BigInt(Math.floor(((nowMs - nowMsInt) * 1_000_000)));
buffer.writeUInt64LE(mem, outPtr, now)

// Success.
return 0;
}
default:
// Return an `EINVAL` error.
return 28
}
},

// Writing to a file descriptor is used in order to write to stdout/stderr.
fd_write: (fd: number, addr: number, num: number, outPtr: number) => {
const instance = state.instance!;

outPtr >>>= 0;

// Only stdout and stderr are open for writing.
if (fd != 1 && fd != 2) {
return 8;
}

const mem = new Uint8Array(instance.exports.memory.buffer);

// `fd_write` passes a buffer containing itself a list of pointers and lengths to the
// actual buffers. See writev(2).
let toWrite = "";
let totalLength = 0;
for (let i = 0; i < num; i++) {
const buf = buffer.readUInt32LE(mem, addr + 4 * i * 2);
const bufLen = buffer.readUInt32LE(mem, addr + 4 * (i * 2 + 1));
toWrite += buffer.utf8BytesToString(mem, buf, bufLen);
totalLength += bufLen;
}

const flushBuffer = (string: string) => {
// As documented in the documentation of `println!`, lines are always split by a
// single `\n` in Rust.
while (true) {
const index = string.indexOf('\n');
if (index != -1) {
// Note that it is questionnable to use `console.log` from within a
// library. However this simply reflects the usage of `println!` in the
// Rust code. In other words, it is `println!` that shouldn't be used in
// the first place. The harm of not showing text printed with `println!`
// at all is greater than the harm possibly caused by accidentally leaving
// a `println!` in the code.
console.log(string.substring(0, index));
string = string.substring(index + 1);
} else {
return string;
}
}
};

// Append the newly-written data to either `stdout_buffer` or `stderr_buffer`, and
// print their content if necessary.
if (fd == 1) {
state.stdoutBuffer += toWrite;
state.stdoutBuffer = flushBuffer(state.stdoutBuffer);
} else if (fd == 2) {
state.stderrBuffer += toWrite;
state.stderrBuffer = flushBuffer(state.stderrBuffer);
}

// Need to write in `out_ptr` how much data was "written".
buffer.writeUInt32LE(mem, outPtr, totalLength);
return 0;
},

// It's unclear how to properly implement yielding, but a no-op works fine as well.
sched_yield: () => {
return 0;
},

// Used by Rust in catastrophic situations, such as a double panic.
proc_exit: (retCode: number) => {
state.instance = null;
eventCallback({
ty: "wasm-panic",
message: `proc_exit called: ${retCode}`,
currentTask: state.currentTask
});
state.onShutdownExecutorOrWasmPanic();
state.onShutdownExecutorOrWasmPanic = () => { };
throw new Error();
},

// Return the number of environment variables and the total size of all environment
// variables. This is called in order to initialize buffers before `environ_get`.
environ_sizes_get: (argcOut: number, argvBufSizeOut: number) => {
const instance = state.instance!;

argcOut >>>= 0;
argvBufSizeOut >>>= 0;

let totalLen = 0;
config.envVars.forEach(e => totalLen += new TextEncoder().encode(e).length + 1); // +1 for trailing \0

const mem = new Uint8Array(instance.exports.memory.buffer);
buffer.writeUInt32LE(mem, argcOut, config.envVars.length);
buffer.writeUInt32LE(mem, argvBufSizeOut, totalLen);
return 0;
},

// Write the environment variables to the given pointers.
// `argv` is a pointer to a buffer that must be overwritten with a list of pointers to
// environment variables, and `argvBuf` is a pointer to a buffer where to actually store
// the environment variables.
// The sizes of the buffers were determined by calling `environ_sizes_get`.
environ_get: (argv: number, argvBuf: number) => {
const instance = state.instance!;

argv >>>= 0;
argvBuf >>>= 0;

const mem = new Uint8Array(instance.exports.memory.buffer);

let argvPos = 0;
let argvBufPos = 0;

config.envVars.forEach(envVar => {
const encoded = new TextEncoder().encode(envVar);

buffer.writeUInt32LE(mem, argv + argvPos, argvBuf + argvBufPos);
argvPos += 4;

mem.set(encoded, argvBuf + argvBufPos);
argvBufPos += encoded.length;
buffer.writeUInt8(mem, argvBuf + argvBufPos, 0);
argvBufPos += 1;
});

return 0;
},
};

// Start the Wasm virtual machine.
// The Rust code defines a list of imports that must be fulfilled by the environment. The second
// parameter provides their implementations.
const result = await WebAssembly.instantiate(wasmModule, {
// The functions with the "smoldot" prefix are specific to smoldot.
"smoldot": smoldotJsBindings,
// As the Rust code is compiled for wasi, some more wasi-specific imports exist.
"wasi_snapshot_preview1": wasiBindings,
});

state.instance = result as SmoldotWasmInstance;
Expand Down
1 change: 0 additions & 1 deletion wasm-node/rust/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ log = { version = "0.4.18", features = ["std"] }
nom = { version = "7.1.3", default-features = false }
no-std-net = { version = "0.6.0", default-features = false }
pin-project = "1.1.3"
rand = "0.8.5"
slab = { version = "0.4.8", default-features = false }
smoldot = { version = "0.12.0", path = "../../lib", default-features = false }
smoldot-light = { version = "0.10.0", path = "../../light-base", default-features = false }
31 changes: 20 additions & 11 deletions wasm-node/rust/src/bindings.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,6 @@
//!
//! This avoids potential stack overflows and tricky borrowing-related situations.
//!
//! # About wasi
//!
//! The Rust code is expected to be compiled for the `wasm32-wasi` target, and not just
//! `wasm32-unknown-unknown`. The `wasi` platform is used in order for example to obtain a source
//! of randomness and time.
//!
//! Consequently, the exports found in the `extern` block below are not the only functions that
//! must be implemented. Several functions required by the Wasi ABI are also used. The best place
//! to find documentation at the moment is <https://docs.rs/wasi>.
//!
//! # About `u32`s and JavaScript
//!
//! Many functions below accept as parameter or return a `u32`. In reality, however, the
Expand Down Expand Up @@ -91,6 +81,25 @@ extern "C" {
/// behave like `abort` and prevent any further execution.
pub fn panic(message_ptr: u32, message_len: u32);

/// Fills the buffer of the WebAssembly virtual machine with random data, starting at `ptr`
/// and for `len` bytes.
///
/// This data will be used in order to generate secrets. Do not use a dummy implementation!
pub fn random_get(ptr: u32, len: u32);

/// Returns the system clock in number of microseconds since the UNIX epoch, ignoring leap
/// seconds.
///
/// This clock is allowed to go backwards.
///
/// Must never return a negative number. Implementers should be aware that the system clock
/// can be negative, and abort execution if that is the case.
pub fn unix_timestamp_us() -> u64;

/// Returns the number of microseconds since an especified point in time. Must never decrease
/// over time.
pub fn monotonic_clock_us() -> u64;

/// Copies the entire content of the buffer with the given index to the memory of the
/// WebAssembly at offset `target_pointer`.
///
Expand Down Expand Up @@ -135,7 +144,7 @@ extern "C" {
/// have passed, and this will likely cause smoldot to restart a new timer for the remainder
/// of the duration.
///
/// When [`timer_finished`] is called, the value of the monotonic clock (in the WASI bindings)
/// When [`timer_finished`] is called, the value of the monotonic clock (in the bindings)
/// must have increased by at least the given number of `milliseconds`.
///
/// If `milliseconds` is 0, [`timer_finished`] should be called as soon as possible.
Expand Down
4 changes: 0 additions & 4 deletions wasm-node/rust/src/init.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,6 @@ pub(crate) fn init(max_log_level: u32) {
env!("CARGO_PKG_VERSION")
);

// Simple fool-proof check to make sure that randomness is properly implemented.
assert_ne!(rand::random::<u64>(), 0);
assert_ne!(rand::random::<u64>(), rand::random::<u64>());

// Spawn a constantly-running task that periodically prints the total memory usage of
// the node.
platform::PLATFORM_REF.spawn_task(
Expand Down
Loading

0 comments on commit d353b6c

Please sign in to comment.