From a649601e5de00b206f9fae51d1b9d1bc84031429 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Mon, 25 Nov 2024 12:02:01 +0100 Subject: [PATCH 01/10] piecrust: added callstack api --- contracts/callcenter/src/lib.rs | 11 +++++++++++ piecrust/src/imports.rs | 16 ++++++++++++++++ piecrust/src/session.rs | 4 ++++ piecrust/tests/callcenter.rs | 24 ++++++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/contracts/callcenter/src/lib.rs b/contracts/callcenter/src/lib.rs index b6801c86..b8339dbb 100644 --- a/contracts/callcenter/src/lib.rs +++ b/contracts/callcenter/src/lib.rs @@ -80,6 +80,11 @@ impl Callcenter { uplink::caller() } + /// Return the entire call stack of this contract + pub fn return_callstack(&self) -> Vec { + uplink::callstack() + } + /// Make sure that the caller of this contract is the contract itself pub fn call_self(&self) -> Result { let self_id = uplink::self_id(); @@ -153,6 +158,12 @@ unsafe fn return_caller(arg_len: u32) -> u32 { wrap_call(arg_len, |_: ()| STATE.return_caller()) } +/// Expose `Callcenter::return_callstack()` to the host +#[no_mangle] +unsafe fn return_callstack(arg_len: u32) -> u32 { + wrap_call(arg_len, |_: ()| STATE.return_callstack()) +} + /// Expose `Callcenter::delegate_query()` to the host #[no_mangle] unsafe fn delegate_query(arg_len: u32) -> u32 { diff --git a/piecrust/src/imports.rs b/piecrust/src/imports.rs index 79a441e4..d8d3c449 100644 --- a/piecrust/src/imports.rs +++ b/piecrust/src/imports.rs @@ -55,6 +55,7 @@ impl Imports { fn import(store: &mut Store, name: &str, is_64: bool) -> Option { Some(match name { "caller" => Func::wrap(store, caller), + "callstack" => Func::wrap(store, callstack), "c" => match is_64 { false => Func::wrap(store, wasm32::c), true => Func::wrap(store, wasm64::c), @@ -386,6 +387,21 @@ fn caller(env: Caller) -> i32 { } } +fn callstack(env: Caller) -> i32 { + let env = env.data(); + let instance = env.self_instance(); + + let mut i = 0usize; + for contract_id in env.callstack_iter() { + instance.with_arg_buf_mut(|buf| { + buf[i * CONTRACT_ID_BYTES..(i + 1) * CONTRACT_ID_BYTES] + .copy_from_slice(contract_id.as_bytes()); + }); + i += 1; + } + i as i32 +} + fn feed(mut fenv: Caller, arg_len: u32) -> WasmtimeResult<()> { let env = fenv.data_mut(); let instance = env.self_instance(); diff --git a/piecrust/src/session.rs b/piecrust/src/session.rs index bd40ce82..a3482fe5 100644 --- a/piecrust/src/session.rs +++ b/piecrust/src/session.rs @@ -643,6 +643,10 @@ impl Session { self.inner.call_tree.nth_parent(n) } + pub(crate) fn callstack_iter(&self) -> impl Iterator { + self.inner.call_tree.iter().map(|elem| &elem.contract_id) + } + /// Creates a new instance of the given contract, returning its memory /// length. fn create_instance( diff --git a/piecrust/tests/callcenter.rs b/piecrust/tests/callcenter.rs index 5d8545b8..bd8801b0 100644 --- a/piecrust/tests/callcenter.rs +++ b/piecrust/tests/callcenter.rs @@ -242,6 +242,30 @@ pub fn cc_caller_uninit() -> Result<(), Error> { Ok(()) } +#[test] +pub fn cc_callstack() -> Result<(), Error> { + let vm = VM::ephemeral()?; + + let mut session = vm.session(SessionData::builder())?; + + let center_id = session.deploy( + contract_bytecode!("callcenter"), + ContractData::builder().owner(OWNER), + LIMIT, + )?; + + let callstack: Vec = session + .call(center_id, "return_callstack", &(), LIMIT)? + .data; + assert_eq!(callstack.len(), 1); + + let self_id: ContractId = + session.call(center_id, "return_self_id", &(), LIMIT)?.data; + assert_eq!(callstack[0], self_id); + + Ok(()) +} + #[test] pub fn cc_self_id() -> Result<(), Error> { let vm = VM::ephemeral()?; From 1f5de1e9a34e35259e48496660e9e13fc13ad5f8 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Mon, 25 Nov 2024 12:02:29 +0100 Subject: [PATCH 02/10] piecrust-uplink: added callstack api --- piecrust-uplink/src/abi/state.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/piecrust-uplink/src/abi/state.rs b/piecrust-uplink/src/abi/state.rs index 5073587b..c5617952 100644 --- a/piecrust-uplink/src/abi/state.rs +++ b/piecrust-uplink/src/abi/state.rs @@ -58,6 +58,7 @@ mod ext { pub fn feed(arg_len: u32); pub fn caller() -> i32; + pub fn callstack() -> i32; pub fn limit() -> u64; pub fn spent() -> u64; pub fn owner(contract_id: *const u8) -> i32; @@ -289,6 +290,22 @@ pub fn caller() -> Option { } } +/// Returns IDs of all calling contracts present in the calling stack +pub fn callstack() -> Vec { + let n = unsafe { ext::callstack() }; + with_arg_buf(|buf| { + let mut v = Vec::new(); + for i in 0..n as usize { + let mut bytes = [0; CONTRACT_ID_BYTES]; + bytes.copy_from_slice( + &buf[i * CONTRACT_ID_BYTES..(i + 1) * CONTRACT_ID_BYTES], + ); + v.push(ContractId::from_bytes(bytes)); + } + v + }) +} + /// Returns the gas limit with which the contact was called. pub fn limit() -> u64 { unsafe { ext::limit() } From f1507feb10678a0b5a307df33ee27db0f5d31d1c Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Mon, 25 Nov 2024 12:55:15 +0100 Subject: [PATCH 03/10] piecrust: improved implementation and test --- contracts/callcenter/src/lib.rs | 16 ++++++++++++++++ piecrust/src/imports.rs | 4 ++-- piecrust/src/session.rs | 4 ---- piecrust/tests/callcenter.rs | 9 +++++++++ 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/contracts/callcenter/src/lib.rs b/contracts/callcenter/src/lib.rs index b8339dbb..4b08ffd4 100644 --- a/contracts/callcenter/src/lib.rs +++ b/contracts/callcenter/src/lib.rs @@ -95,6 +95,16 @@ impl Callcenter { } } + /// Return a call stack after calling itself n times + pub fn call_self_n_times(&self, n: u32) -> Vec { + let self_id = uplink::self_id(); + match n { + 0 => uplink::callstack(), + _ => uplink::call(self_id, "call_self_n_times", &(n - 1)) + .expect("calling self should succeed") + } + } + /// Calls the `spend` function of the `contract` with no arguments, and the /// given `gas_limit`, assuming the called function returns `()`. It will /// then return the call's result itself. @@ -138,6 +148,12 @@ unsafe fn call_self(arg_len: u32) -> u32 { wrap_call(arg_len, |_: ()| STATE.call_self()) } +/// Expose `Callcenter::call_self_n_times()` to the host +#[no_mangle] +unsafe fn call_self_n_times(arg_len: u32) -> u32 { + wrap_call(arg_len, |n: u32| STATE.call_self_n_times(n)) +} + /// Expose `Callcenter::call_spend_with_limit` to the host #[no_mangle] unsafe fn call_spend_with_limit(arg_len: u32) -> u32 { diff --git a/piecrust/src/imports.rs b/piecrust/src/imports.rs index d8d3c449..d483db65 100644 --- a/piecrust/src/imports.rs +++ b/piecrust/src/imports.rs @@ -392,10 +392,10 @@ fn callstack(env: Caller) -> i32 { let instance = env.self_instance(); let mut i = 0usize; - for contract_id in env.callstack_iter() { + while let Some(element) = env.nth_from_top(i) { instance.with_arg_buf_mut(|buf| { buf[i * CONTRACT_ID_BYTES..(i + 1) * CONTRACT_ID_BYTES] - .copy_from_slice(contract_id.as_bytes()); + .copy_from_slice(element.contract_id.as_bytes()); }); i += 1; } diff --git a/piecrust/src/session.rs b/piecrust/src/session.rs index a3482fe5..bd40ce82 100644 --- a/piecrust/src/session.rs +++ b/piecrust/src/session.rs @@ -643,10 +643,6 @@ impl Session { self.inner.call_tree.nth_parent(n) } - pub(crate) fn callstack_iter(&self) -> impl Iterator { - self.inner.call_tree.iter().map(|elem| &elem.contract_id) - } - /// Creates a new instance of the given contract, returning its memory /// length. fn create_instance( diff --git a/piecrust/tests/callcenter.rs b/piecrust/tests/callcenter.rs index bd8801b0..c5d9b05b 100644 --- a/piecrust/tests/callcenter.rs +++ b/piecrust/tests/callcenter.rs @@ -263,6 +263,15 @@ pub fn cc_callstack() -> Result<(), Error> { session.call(center_id, "return_self_id", &(), LIMIT)?.data; assert_eq!(callstack[0], self_id); + const N: u32 = 5; + let callstack: Vec = session + .call(center_id, "call_self_n_times", &N, LIMIT)? + .data; + assert_eq!(callstack.len(), N as usize + 1); + for i in 1..=N as usize { + assert_eq!(callstack[0], callstack[i]); + } + Ok(()) } From 34b2bdfed5094f6b29f06487d574719426f95c84 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Mon, 25 Nov 2024 14:47:14 +0100 Subject: [PATCH 04/10] piecrust: optimized callstack implementation --- contracts/Cargo.toml | 1 + contracts/callstack/Cargo.toml | 15 +++++++++++++++ contracts/callstack/src/lib.rs | 34 ++++++++++++++++++++++++++++++++++ piecrust/src/call_tree.rs | 14 ++++++++++++++ piecrust/src/imports.rs | 4 ++-- piecrust/src/session.rs | 4 ++++ piecrust/tests/callcenter.rs | 27 +++++++++++++++++++++++++++ 7 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 contracts/callstack/Cargo.toml create mode 100644 contracts/callstack/src/lib.rs diff --git a/contracts/Cargo.toml b/contracts/Cargo.toml index af233596..d03b0729 100644 --- a/contracts/Cargo.toml +++ b/contracts/Cargo.toml @@ -3,6 +3,7 @@ members = [ "box", "c-example", "callcenter", + "callstack", "initializer", "counter", "counter_float", diff --git a/contracts/callstack/Cargo.toml b/contracts/callstack/Cargo.toml new file mode 100644 index 00000000..9a0b8a1a --- /dev/null +++ b/contracts/callstack/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "callstack" +version = "0.1.0" +authors = [ + "Milosz Muszynski ", +] +edition = "2021" + +license = "MPL-2.0" + +[dependencies] +piecrust-uplink = { path = "../../piecrust-uplink", features = ["abi", "dlmalloc"] } + +[lib] +crate-type = ["cdylib", "rlib"] diff --git a/contracts/callstack/src/lib.rs b/contracts/callstack/src/lib.rs new file mode 100644 index 00000000..c4239747 --- /dev/null +++ b/contracts/callstack/src/lib.rs @@ -0,0 +1,34 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// Copyright (c) DUSK NETWORK. All rights reserved. + +//! Contract which exposes the call stack + +#![no_std] + +extern crate alloc; + +use piecrust_uplink as uplink; +use alloc::vec::Vec; +use uplink::ContractId; + +/// Struct that describes the state of the contract +pub struct CallStack; + +/// State of the Counter contract +static mut STATE: CallStack = CallStack; + +impl CallStack { + /// Return the call stack + pub fn return_callstack(&self) -> Vec { + uplink::callstack() + } +} + +/// Expose `CallStack::read_callstack()` to the host +#[no_mangle] +unsafe fn return_callstack(arg_len: u32) -> u32 { + uplink::wrap_call_unchecked(arg_len, |_: ()| STATE.return_callstack()) +} diff --git a/piecrust/src/call_tree.rs b/piecrust/src/call_tree.rs index 15a5d1be..b578a0b3 100644 --- a/piecrust/src/call_tree.rs +++ b/piecrust/src/call_tree.rs @@ -101,6 +101,20 @@ impl CallTree { current.map(|inner| unsafe { (*inner).elem }) } + /// Returns all call ids. + pub(crate) fn call_ids(&self) -> Vec<&ContractId> { + let mut v = Vec::new(); + let mut current = self.0; + + while current.is_some() { + let p = *current.as_ref().unwrap(); + v.push(unsafe { &(*p).elem.contract_id }); + current = current.and_then(|inner| unsafe { (*inner).parent }); + } + + v + } + /// Clears the call tree of all elements. pub(crate) fn clear(&mut self) { unsafe { diff --git a/piecrust/src/imports.rs b/piecrust/src/imports.rs index d483db65..dd5d7cb0 100644 --- a/piecrust/src/imports.rs +++ b/piecrust/src/imports.rs @@ -392,10 +392,10 @@ fn callstack(env: Caller) -> i32 { let instance = env.self_instance(); let mut i = 0usize; - while let Some(element) = env.nth_from_top(i) { + for contract_id in env.call_ids() { instance.with_arg_buf_mut(|buf| { buf[i * CONTRACT_ID_BYTES..(i + 1) * CONTRACT_ID_BYTES] - .copy_from_slice(element.contract_id.as_bytes()); + .copy_from_slice(contract_id.as_bytes()); }); i += 1; } diff --git a/piecrust/src/session.rs b/piecrust/src/session.rs index bd40ce82..6e90399e 100644 --- a/piecrust/src/session.rs +++ b/piecrust/src/session.rs @@ -643,6 +643,10 @@ impl Session { self.inner.call_tree.nth_parent(n) } + pub(crate) fn call_ids(&self) -> Vec<&ContractId> { + self.inner.call_tree.call_ids() + } + /// Creates a new instance of the given contract, returning its memory /// length. fn create_instance( diff --git a/piecrust/tests/callcenter.rs b/piecrust/tests/callcenter.rs index c5d9b05b..e9fcb21f 100644 --- a/piecrust/tests/callcenter.rs +++ b/piecrust/tests/callcenter.rs @@ -254,6 +254,12 @@ pub fn cc_callstack() -> Result<(), Error> { LIMIT, )?; + let callstack_id = session.deploy( + contract_bytecode!("callstack"), + ContractData::builder().owner(OWNER), + LIMIT, + )?; + let callstack: Vec = session .call(center_id, "return_callstack", &(), LIMIT)? .data; @@ -272,6 +278,27 @@ pub fn cc_callstack() -> Result<(), Error> { assert_eq!(callstack[0], callstack[i]); } + let res = session + .call::<_, Result, ContractError>>( + center_id, + "delegate_query", + &( + callstack_id, + String::from("return_callstack"), + Vec::::new(), + ), + LIMIT, + )? + .data + .expect("ICC should succeed"); + + let callstack: Vec = + rkyv::from_bytes(&res).expect("Deserialization to succeed"); + + assert_eq!(callstack.len(), 2); + assert_eq!(callstack[0], callstack_id); + assert_eq!(callstack[1], center_id); + Ok(()) } From 3719dbf2fadf5eb37863d6e0d207255f22329a71 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Tue, 26 Nov 2024 10:56:15 +0100 Subject: [PATCH 05/10] piecrust: bumped version to 0.27.0-rc.0 --- piecrust/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piecrust/Cargo.toml b/piecrust/Cargo.toml index 701df91e..c2b41077 100644 --- a/piecrust/Cargo.toml +++ b/piecrust/Cargo.toml @@ -7,7 +7,7 @@ categories = ["wasm", "no-std", "cryptography::cryptocurrencies"] keywords = ["virtual", "machine", "smart", "contract", "wasm"] repository = "https://github.com/dusk-network/piecrust" -version = "0.26.1-rc.3" +version = "0.27.0-rc.0" edition = "2021" license = "MPL-2.0" From 23c634aa3d49e26705f763f9e196f7153edefb3f Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Tue, 26 Nov 2024 11:00:24 +0100 Subject: [PATCH 06/10] piecrust-uplink: bumped version to 0.17.2-rc.0 --- piecrust-uplink/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piecrust-uplink/Cargo.toml b/piecrust-uplink/Cargo.toml index fc4b7a68..76cd6072 100644 --- a/piecrust-uplink/Cargo.toml +++ b/piecrust-uplink/Cargo.toml @@ -7,7 +7,7 @@ categories = ["wasm", "no-std", "cryptography::cryptocurrencies"] keywords = ["virtual", "machine", "smart", "contract", "wasm"] repository = "https://github.com/dusk-network/piecrust" -version = "0.17.1" +version = "0.17.2-rc.0" edition = "2021" license = "MPL-2.0" From b41d2055d1df8be3d9eb112c91aa33e7ebe4405c Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Tue, 26 Nov 2024 11:01:12 +0100 Subject: [PATCH 07/10] piecrust: updated uplink dependency to 0.17.2-rc.0 --- piecrust/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piecrust/Cargo.toml b/piecrust/Cargo.toml index c2b41077..7b4d3763 100644 --- a/piecrust/Cargo.toml +++ b/piecrust/Cargo.toml @@ -14,7 +14,7 @@ license = "MPL-2.0" [dependencies] crumbles = { version = "0.3", path = "../crumbles" } -piecrust-uplink = { version = "0.17", path = "../piecrust-uplink" } +piecrust-uplink = { version = "0.17.2-rc.0", path = "../piecrust-uplink" } dusk-wasmtime = { version = "21.0.0-alpha", default-features = false, features = ["cranelift", "runtime", "parallel-compilation"] } bytecheck = "0.6" From 9b5b43ecdf9cbc56bc2eab827a234dfb3b0e3fdd Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Thu, 28 Nov 2024 15:11:01 +0100 Subject: [PATCH 08/10] piecrust: added test for ICCs not rolled back when panic --- contracts/crossover/src/lib.rs | 29 +++++++++++++++++++++++++ piecrust/tests/crossover.rs | 39 ++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/contracts/crossover/src/lib.rs b/contracts/crossover/src/lib.rs index a10f74ef..9729e2e7 100644 --- a/contracts/crossover/src/lib.rs +++ b/contracts/crossover/src/lib.rs @@ -67,6 +67,27 @@ impl Crossover { self.set_crossover(value_to_set); } + // Chain of ICC is not being rolled back when a callee panics and its panic + // is not propagated up the call chain. + pub fn check_iccs_dont_rollback( + &mut self, + contract: ContractId, + value_to_set: i32, + ) { + self.set_crossover(value_to_set); + + const ANY_VALUE_1: i32 = 5; + const ANY_VALUE_2: i32 = 6; + + uplink::debug!("calling panicking contract {contract:?}"); + uplink::call::<_, ()>( + contract, + "set_back_and_panic", + &(ANY_VALUE_1, ANY_VALUE_2), + ) + .expect_err("should give an error on a panic"); + } + // Sets the contract's value and then calls its caller's [`set_crossover`] // call to set their value. The caller is assumed to be another crossover // contract. @@ -125,6 +146,14 @@ unsafe fn check_consistent_state_on_errors(arg_len: u32) -> u32 { }) } +/// Expose `Crossover::check_iccs_dont_rollback()` to the host +#[no_mangle] +unsafe fn check_iccs_dont_rollback(arg_len: u32) -> u32 { + uplink::wrap_call(arg_len, |(contract, s)| { + STATE.check_iccs_dont_rollback(contract, s) + }) +} + /// Expose `Crossover::set_back_and_panic()` to the host #[no_mangle] unsafe fn set_back_and_panic(arg_len: u32) -> u32 { diff --git a/piecrust/tests/crossover.rs b/piecrust/tests/crossover.rs index 97a947c1..ab475886 100644 --- a/piecrust/tests/crossover.rs +++ b/piecrust/tests/crossover.rs @@ -70,3 +70,42 @@ fn crossover() -> Result<(), Error> { Ok(()) } + +#[test] +fn iccs_dont_rollback() -> Result<(), Error> { + let vm = VM::ephemeral()?; + + let mut session = vm.session(SessionData::builder())?; + + session.deploy( + contract_bytecode!("crossover"), + ContractData::builder() + .owner(OWNER) + .contract_id(CROSSOVER_ONE), + LIMIT, + )?; + session.deploy( + contract_bytecode!("crossover"), + ContractData::builder() + .owner(OWNER) + .contract_id(CROSSOVER_TWO), + LIMIT, + )?; + // These value should not be set to `INITIAL_VALUE` in the contract. + const CROSSOVER_TO_SET: i32 = 42; + + session.call::<_, ()>( + CROSSOVER_ONE, + "check_iccs_dont_rollback", + &(CROSSOVER_TWO, CROSSOVER_TO_SET), + LIMIT, + )?; + + assert_eq!( + session.call::<_, i32>(CROSSOVER_ONE, "crossover", &(), LIMIT)?.data, + CROSSOVER_TO_SET, + "The crossover should still be set even though the other contract panicked" + ); + + Ok(()) +} From d7f859777ee56fe7bbe275f75f00ee02ffa79346 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Mon, 2 Dec 2024 13:10:00 +0100 Subject: [PATCH 09/10] piecrust: optimisation of the merkle position file --- piecrust/src/store.rs | 66 +++++++++++++++++++++------- piecrust/src/store/tree.rs | 88 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 134 insertions(+), 20 deletions(-) diff --git a/piecrust/src/store.rs b/piecrust/src/store.rs index 077dbd1a..1fa603e4 100644 --- a/piecrust/src/store.rs +++ b/piecrust/src/store.rs @@ -19,9 +19,11 @@ use std::collections::btree_map::Entry::*; use std::collections::btree_map::Keys; use std::collections::{BTreeMap, BTreeSet}; use std::fmt::{Debug, Formatter}; -use std::fs::create_dir_all; +use std::fs::{create_dir_all, OpenOptions}; +use std::io::{BufReader, BufWriter}; use std::path::{Path, PathBuf}; use std::sync::{mpsc, Arc, Mutex}; +use std::time::SystemTime; use std::{fs, io, thread}; use dusk_wasmtime::Engine; @@ -46,6 +48,7 @@ const MEMORY_DIR: &str = "memory"; const LEAF_DIR: &str = "leaf"; const BASE_FILE: &str = "base"; const TREE_POS_FILE: &str = "tree_pos"; +const TREE_POS_OPT_FILE: &str = "tree_pos_opt"; const ELEMENT_FILE: &str = "element"; const OBJECTCODE_EXTENSION: &str = "a"; const METADATA_EXTENSION: &str = "m"; @@ -367,11 +370,11 @@ fn base_path_main, S: AsRef>( fn tree_pos_path_main, S: AsRef>( main_dir: P, commit_id: S, -) -> io::Result { +) -> io::Result<(PathBuf, PathBuf)> { let commit_id = commit_id.as_ref(); let dir = main_dir.as_ref().join(commit_id); fs::create_dir_all(&dir)?; - Ok(dir.join(TREE_POS_FILE)) + Ok((dir.join(TREE_POS_FILE), dir.join(TREE_POS_OPT_FILE))) } fn commit_id_to_hash>(commit_id: S) -> Hash { @@ -422,7 +425,8 @@ fn commit_from_dir>( let tree_pos = if let Some(ref hash_hex) = commit_id { let tree_pos_path = main_dir.join(hash_hex).join(TREE_POS_FILE); - Some(tree_pos_from_path(tree_pos_path)?.tree_pos) + let tree_pos_opt_path = main_dir.join(hash_hex).join(TREE_POS_OPT_FILE); + Some(tree_pos_from_path(tree_pos_path, tree_pos_opt_path)?) } else { None }; @@ -510,7 +514,7 @@ fn index_merkle_from_path( leaf_dir: impl AsRef, maybe_commit_id: &Option, commit_store: Arc>, - maybe_tree_pos: Option<&BTreeMap>, + maybe_tree_pos: Option<&TreePos>, ) -> io::Result<(NewContractIndex, ContractsMerkle)> { let leaf_dir = leaf_dir.as_ref(); @@ -586,18 +590,27 @@ fn base_from_path>(path: P) -> io::Result { Ok(base_info) } -fn tree_pos_from_path>(path: P) -> io::Result { +fn tree_pos_from_path( + path: impl AsRef, + opt_path: impl AsRef, +) -> io::Result { let path = path.as_ref(); - let tree_pos_bytes = fs::read(path)?; - let tree_pos = rkyv::from_bytes(&tree_pos_bytes).map_err(|err| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Invalid tree positions file \"{path:?}\": {err}"), - ) - })?; + let tree_pos = if opt_path.as_ref().exists() { + let f = OpenOptions::new().read(true).open(opt_path.as_ref())?; + let mut buf_f = BufReader::new(f); + TreePos::unmarshall(&mut buf_f) + } else { + let tree_pos_bytes = fs::read(path)?; + rkyv::from_bytes(&tree_pos_bytes).map_err(|err| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid tree positions file \"{path:?}\": {err}"), + ) + }) + }; - Ok(tree_pos) + tree_pos } #[derive(Debug, Clone)] @@ -1107,8 +1120,9 @@ fn write_commit_inner, S: AsRef>( })?; fs::write(base_main_path, base_info_bytes)?; - let tree_pos_main_path = + let (tree_pos_main_path, tree_pos_opt_path) = tree_pos_path_main(&directories.main_dir, commit_id.as_ref())?; + let start = SystemTime::now(); let tree_pos_bytes = rkyv::to_bytes::<_, 128>( commit.contracts_merkle.tree_pos(), ) @@ -1118,7 +1132,25 @@ fn write_commit_inner, S: AsRef>( format!("Failed serializing tree positions file: {err}"), ) })?; - fs::write(tree_pos_main_path, tree_pos_bytes)?; + fs::write(tree_pos_main_path.clone(), tree_pos_bytes)?; + let stop = SystemTime::now(); + println!( + "WRITE TREE POS RKYV FINISHED, ELAPSED TIME={:?}", + stop.duration_since(start).expect("duration should work") + ); + + let start = SystemTime::now(); + let f = OpenOptions::new() + .append(true) + .create(true) + .open(tree_pos_opt_path)?; + let mut buf_f = BufWriter::new(f); + commit.contracts_merkle.tree_pos().marshall(&mut buf_f)?; + let stop = SystemTime::now(); + println!( + "WRITE TREE POS BINARY FINISHED, ELAPSED TIME={:?}", + stop.duration_since(start).expect("duration should work") + ); Ok(()) } @@ -1161,6 +1193,7 @@ fn finalize_commit>( let commit_path = main_dir.join(&root); let base_info_path = commit_path.join(BASE_FILE); let tree_pos_path = commit_path.join(TREE_POS_FILE); + let tree_pos_opt_path = commit_path.join(TREE_POS_OPT_FILE); let base_info = base_from_path(&base_info_path)?; for contract_hint in base_info.contract_hints { let contract_hex = hex::encode(contract_hint); @@ -1191,6 +1224,7 @@ fn finalize_commit>( fs::remove_file(base_info_path)?; fs::remove_file(tree_pos_path)?; + fs::remove_file(tree_pos_opt_path)?; fs::remove_dir(commit_path)?; Ok(()) diff --git a/piecrust/src/store/tree.rs b/piecrust/src/store/tree.rs index 96e5d1ee..123f37ad 100644 --- a/piecrust/src/store/tree.rs +++ b/piecrust/src/store/tree.rs @@ -12,6 +12,7 @@ use std::{ use bytecheck::CheckBytes; use piecrust_uplink::ContractId; use rkyv::{Archive, Deserialize, Serialize}; +use std::io::{self, ErrorKind, Read, Write}; // There are max `2^16` pages in a 32-bit memory const P32_HEIGHT: usize = 8; @@ -97,7 +98,7 @@ impl NewContractIndex { pub struct ContractsMerkle { inner_tree: Tree, dict: BTreeMap, - tree_pos: BTreeMap, + tree_pos: TreePos, } impl Default for ContractsMerkle { @@ -105,7 +106,7 @@ impl Default for ContractsMerkle { Self { inner_tree: Tree::new(), dict: BTreeMap::new(), - tree_pos: BTreeMap::new(), + tree_pos: TreePos::default(), } } } @@ -140,7 +141,7 @@ impl ContractsMerkle { self.inner_tree.root() } - pub fn tree_pos(&self) -> &BTreeMap { + pub fn tree_pos(&self) -> &TreePos { &self.tree_pos } @@ -168,7 +169,63 @@ pub struct BaseInfo { #[derive(Debug, Clone, Default, Archive, Deserialize, Serialize)] #[archive_attr(derive(CheckBytes))] pub struct TreePos { - pub tree_pos: BTreeMap, + tree_pos: BTreeMap, +} + +impl TreePos { + pub fn insert(&mut self, k: u32, v: (Hash, u64)) { + self.tree_pos.insert(k, v); + } + + pub fn marshall(&self, w: &mut W) -> io::Result<()> { + for (k, (h, p)) in self.tree_pos.iter() { + w.write_all(&(*k).to_le_bytes())?; + w.write_all(h.as_bytes())?; + w.write_all(&(*p as u32).to_le_bytes())?; + } + Ok(()) + } + + pub fn unmarshall(r: &mut R) -> io::Result { + let mut slf = Self::default(); + loop { + let mut buf_k = [0u8; 4]; + let e = r.read_exact(&mut buf_k); + if e.as_ref() + .is_err_and(|e| e.kind() == ErrorKind::UnexpectedEof) + { + break; + } + e?; + let k = u32::from_le_bytes(buf_k); + + let mut buf_h = [0u8; 32]; + let e = r.read_exact(&mut buf_h); + if e.as_ref() + .is_err_and(|e| e.kind() == ErrorKind::UnexpectedEof) + { + break; + } + e?; + let hash = Hash::from(buf_h); + + let mut buf_p = [0u8; 4]; + let e = r.read_exact(&mut buf_p); + if e.as_ref() + .is_err_and(|e| e.kind() == ErrorKind::UnexpectedEof) + { + break; + } + e?; + let p = u32::from_le_bytes(buf_p); + slf.tree_pos.insert(k, (hash, p as u64)); + } + Ok(slf) + } + + pub fn iter(&self) -> impl Iterator { + self.tree_pos.iter() + } } #[derive(Debug, Clone, Archive, Deserialize, Serialize)] @@ -478,3 +535,26 @@ pub fn position_from_contract(contract: &ContractId) -> u64 { pos as u64 } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{BufReader, BufWriter}; + + #[test] + fn merkle_position_serialization() -> Result<(), io::Error> { + let mut marshalled = TreePos::default(); + let h1 = Hash::from([1u8; 32]); + let h2 = Hash::from([2u8; 32]); + marshalled.insert(2, (h1, 3)); + marshalled.insert(4, (h2, 5)); + let v: Vec = Vec::new(); + let mut w = BufWriter::new(v); + marshalled.marshall(&mut w)?; + let mut r = BufReader::new(w.buffer()); + let unmarshalled = TreePos::unmarshall(&mut r)?; + assert_eq!(unmarshalled.tree_pos.get(&2), Some(&(h1, 3))); + assert_eq!(unmarshalled.tree_pos.get(&4), Some(&(h2, 5))); + Ok(()) + } +} From 2749b8e7d9539ea37880bcdb3cc99bbf06849c63 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Mon, 2 Dec 2024 17:03:53 +0100 Subject: [PATCH 10/10] piecrust: chunking for writing merkle position file --- piecrust/src/store.rs | 35 +++------------- piecrust/src/store/tree.rs | 85 +++++++++++++++++++++++--------------- 2 files changed, 58 insertions(+), 62 deletions(-) diff --git a/piecrust/src/store.rs b/piecrust/src/store.rs index 1fa603e4..4d74c77a 100644 --- a/piecrust/src/store.rs +++ b/piecrust/src/store.rs @@ -23,7 +23,6 @@ use std::fs::{create_dir_all, OpenOptions}; use std::io::{BufReader, BufWriter}; use std::path::{Path, PathBuf}; use std::sync::{mpsc, Arc, Mutex}; -use std::time::SystemTime; use std::{fs, io, thread}; use dusk_wasmtime::Engine; @@ -370,11 +369,11 @@ fn base_path_main, S: AsRef>( fn tree_pos_path_main, S: AsRef>( main_dir: P, commit_id: S, -) -> io::Result<(PathBuf, PathBuf)> { +) -> io::Result { let commit_id = commit_id.as_ref(); let dir = main_dir.as_ref().join(commit_id); fs::create_dir_all(&dir)?; - Ok((dir.join(TREE_POS_FILE), dir.join(TREE_POS_OPT_FILE))) + Ok(dir.join(TREE_POS_OPT_FILE)) } fn commit_id_to_hash>(commit_id: S) -> Hash { @@ -1120,37 +1119,15 @@ fn write_commit_inner, S: AsRef>( })?; fs::write(base_main_path, base_info_bytes)?; - let (tree_pos_main_path, tree_pos_opt_path) = + let tree_pos_opt_path = tree_pos_path_main(&directories.main_dir, commit_id.as_ref())?; - let start = SystemTime::now(); - let tree_pos_bytes = rkyv::to_bytes::<_, 128>( - commit.contracts_merkle.tree_pos(), - ) - .map_err(|err| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Failed serializing tree positions file: {err}"), - ) - })?; - fs::write(tree_pos_main_path.clone(), tree_pos_bytes)?; - let stop = SystemTime::now(); - println!( - "WRITE TREE POS RKYV FINISHED, ELAPSED TIME={:?}", - stop.duration_since(start).expect("duration should work") - ); - - let start = SystemTime::now(); + let f = OpenOptions::new() .append(true) .create(true) .open(tree_pos_opt_path)?; let mut buf_f = BufWriter::new(f); commit.contracts_merkle.tree_pos().marshall(&mut buf_f)?; - let stop = SystemTime::now(); - println!( - "WRITE TREE POS BINARY FINISHED, ELAPSED TIME={:?}", - stop.duration_since(start).expect("duration should work") - ); Ok(()) } @@ -1223,8 +1200,8 @@ fn finalize_commit>( } fs::remove_file(base_info_path)?; - fs::remove_file(tree_pos_path)?; - fs::remove_file(tree_pos_opt_path)?; + let _ = fs::remove_file(tree_pos_path); + let _ = fs::remove_file(tree_pos_opt_path); fs::remove_dir(commit_path)?; Ok(()) diff --git a/piecrust/src/store/tree.rs b/piecrust/src/store/tree.rs index 123f37ad..418e4d84 100644 --- a/piecrust/src/store/tree.rs +++ b/piecrust/src/store/tree.rs @@ -178,46 +178,62 @@ impl TreePos { } pub fn marshall(&self, w: &mut W) -> io::Result<()> { + const CHUNK_SIZE: usize = 8192; + const ELEM_SIZE: usize = 4 + 32 + 4; + let mut b = [0u8; ELEM_SIZE * CHUNK_SIZE]; + let mut chk = 0; for (k, (h, p)) in self.tree_pos.iter() { - w.write_all(&(*k).to_le_bytes())?; - w.write_all(h.as_bytes())?; - w.write_all(&(*p as u32).to_le_bytes())?; + let offset = chk * ELEM_SIZE; + b[offset..(offset + 4)].copy_from_slice(&(*k).to_le_bytes()); + b[(offset + 4)..(offset + 36)].copy_from_slice(h.as_bytes()); + b[(offset + 36)..(offset + 40)] + .copy_from_slice(&(*p as u32).to_le_bytes()); + chk = (chk + 1) % CHUNK_SIZE; + if chk == 0 { + w.write_all(b.as_slice())?; + } + } + if chk != 0 { + w.write_all(&b[..(chk * ELEM_SIZE)])?; } Ok(()) } + fn read_bytes(r: &mut R) -> io::Result<[u8; N]> { + let mut buffer = [0u8; N]; + r.read_exact(&mut buffer)?; + Ok(buffer) + } + + fn is_eof(r: &io::Result) -> bool { + if let Err(ref e) = r { + if e.kind() == ErrorKind::UnexpectedEof { + return true; + } + } + false + } + pub fn unmarshall(r: &mut R) -> io::Result { let mut slf = Self::default(); loop { - let mut buf_k = [0u8; 4]; - let e = r.read_exact(&mut buf_k); - if e.as_ref() - .is_err_and(|e| e.kind() == ErrorKind::UnexpectedEof) - { + let res = Self::read_bytes(r); + if Self::is_eof(&res) { break; } - e?; - let k = u32::from_le_bytes(buf_k); - - let mut buf_h = [0u8; 32]; - let e = r.read_exact(&mut buf_h); - if e.as_ref() - .is_err_and(|e| e.kind() == ErrorKind::UnexpectedEof) - { + let k = u32::from_le_bytes(res?); + + let res = Self::read_bytes(r); + if Self::is_eof(&res) { break; } - e?; - let hash = Hash::from(buf_h); - - let mut buf_p = [0u8; 4]; - let e = r.read_exact(&mut buf_p); - if e.as_ref() - .is_err_and(|e| e.kind() == ErrorKind::UnexpectedEof) - { + let hash = Hash::from(res?); + + let res = Self::read_bytes(r); + if Self::is_eof(&res) { break; } - e?; - let p = u32::from_le_bytes(buf_p); + let p = u32::from_le_bytes(res?); slf.tree_pos.insert(k, (hash, p as u64)); } Ok(slf) @@ -543,18 +559,21 @@ mod tests { #[test] fn merkle_position_serialization() -> Result<(), io::Error> { + const TEST_SIZE: u32 = 262144; + const ELEM_SIZE: usize = 4 + 32 + 4; let mut marshalled = TreePos::default(); - let h1 = Hash::from([1u8; 32]); - let h2 = Hash::from([2u8; 32]); - marshalled.insert(2, (h1, 3)); - marshalled.insert(4, (h2, 5)); + let h = Hash::from([1u8; 32]); + for i in 0..TEST_SIZE { + marshalled.insert(i, (h, i as u64)); + } let v: Vec = Vec::new(); - let mut w = BufWriter::new(v); + let mut w = BufWriter::with_capacity(TEST_SIZE as usize * ELEM_SIZE, v); marshalled.marshall(&mut w)?; let mut r = BufReader::new(w.buffer()); let unmarshalled = TreePos::unmarshall(&mut r)?; - assert_eq!(unmarshalled.tree_pos.get(&2), Some(&(h1, 3))); - assert_eq!(unmarshalled.tree_pos.get(&4), Some(&(h2, 5))); + for i in 0..TEST_SIZE { + assert_eq!(unmarshalled.tree_pos.get(&i), Some(&(h, i as u64))); + } Ok(()) } }