From 36504f49b81ea0bdc58cad8e9009a6736add8af7 Mon Sep 17 00:00:00 2001 From: "Brandon H. Gomes" Date: Tue, 3 May 2022 16:21:04 -0400 Subject: [PATCH] feat: add ledger sync protocol Signed-off-by: Brandon H. Gomes --- pallets/manta-pay/src/lib.rs | 192 +++++++++++++++++++++++---------- pallets/manta-pay/src/types.rs | 13 +-- 2 files changed, 142 insertions(+), 63 deletions(-) diff --git a/pallets/manta-pay/src/lib.rs b/pallets/manta-pay/src/lib.rs index 788e88df6..e9c16212c 100644 --- a/pallets/manta-pay/src/lib.rs +++ b/pallets/manta-pay/src/lib.rs @@ -71,7 +71,7 @@ use manta_crypto::{ constraint::ProofSystem, merkle_tree::{self, forest::Configuration as _}, }; -use manta_pay::config; +use manta_pay::{config, signer::Checkpoint}; use manta_primitives::{ assets::{AssetConfig, FungibleLedger as _, FungibleLedgerError}, types::{AssetId, Balance}, @@ -132,40 +132,29 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet {} - /// Shards of the merkle tree of UTXOs + /// UTXOs and Encrypted Notes Grouped by Shard #[pallet::storage] pub(super) type Shards = StorageDoubleMap<_, Identity, u8, Identity, u64, ([u8; 32], EncryptedNote), ValueQuery>; - /// Shard merkle trees + /// Shard Merkle Tree Paths #[pallet::storage] pub(super) type ShardTrees = StorageMap<_, Identity, u8, UtxoMerkleTreePath, ValueQuery>; - /// Outputs of Utxo accumulator + /// Outputs of Utxo Accumulator #[pallet::storage] pub(super) type UtxoAccumulatorOutputs = StorageMap<_, Identity, [u8; 32], (), ValueQuery>; - /// Utxo set of MantaPay protocol + /// UTXO Set #[pallet::storage] pub(super) type UtxoSet = StorageMap<_, Identity, [u8; 32], (), ValueQuery>; - /// Void number set + /// Void Number Set #[pallet::storage] pub(super) type VoidNumberSet = StorageMap<_, Identity, [u8; 32], (), ValueQuery>; - /// Void number set insertion order - /// Each element of the key is an `u64` insertion order number of void number. - #[pallet::storage] - pub(super) type VoidNumberSetInsertionOrder = - StorageMap<_, Identity, u64, [u8; 32], ValueQuery>; - - /// The size of Void Number Set - /// FIXME: this should be removed. - #[pallet::storage] - pub(super) type VoidNumberSetSize = StorageValue<_, u64, ValueQuery>; - #[pallet::call] impl Pallet { /// Transforms some public assets into private ones using `post`, withdrawing the public @@ -174,15 +163,7 @@ pub mod pallet { #[transactional] pub fn to_private(origin: OriginFor, post: TransferPost) -> DispatchResultWithPostInfo { let origin = ensure_signed(origin)?; - let mut ledger = Self::ledger(); - Self::deposit_event( - config::TransferPost::try_from(post) - .map_err(|_| Error::::InvalidSerializedForm)? - .post(vec![origin], vec![], &(), &mut ledger) - .map_err(Error::::from)? - .convert(None), - ); - Ok(().into()) + Self::post_transaction(None, vec![origin], vec![], post) } /// Transforms some private assets into public ones using `post`, depositing the public @@ -191,15 +172,7 @@ pub mod pallet { #[transactional] pub fn to_public(origin: OriginFor, post: TransferPost) -> DispatchResultWithPostInfo { let origin = ensure_signed(origin)?; - let mut ledger = Self::ledger(); - Self::deposit_event( - config::TransferPost::try_from(post) - .map_err(|_| Error::::InvalidSerializedForm)? - .post(vec![], vec![origin], &(), &mut ledger) - .map_err(Error::::from)? - .convert(None), - ); - Ok(().into()) + Self::post_transaction(None, vec![], vec![origin], post) } /// Transfers private assets encoded in `post`. @@ -215,15 +188,7 @@ pub mod pallet { post: TransferPost, ) -> DispatchResultWithPostInfo { let origin = ensure_signed(origin)?; - let mut ledger = Self::ledger(); - Self::deposit_event( - config::TransferPost::try_from(post) - .map_err(|_| Error::::InvalidSerializedForm)? - .post(vec![], vec![], &(), &mut ledger) - .map_err(Error::::from)? - .convert(Some(origin)), - ); - Ok(().into()) + Self::post_transaction(Some(origin), vec![], vec![], post) } /// Transfers public `asset` from `origin` to the `sink` account. @@ -480,19 +445,140 @@ pub mod pallet { where T: Config, { - /// Returns the ledger implementation for this pallet. + /// Maximum Number of Updates per Shard + const PULL_MAX_PER_SHARD_UPDATE_SIZE: usize = 128; + + /// Maximum Size of Sender Data Update + const PULL_MAX_SENDER_UPDATE_SIZE: usize = 1024; + + /// Pulls receiver data from the ledger starting at the `receiver_index`. + #[inline] + fn pull_receivers( + receiver_index: &mut [usize; 256], + ) -> Result<(bool, ReceiverChunk), scale_codec::Error> { + let mut more_receivers = false; + let mut receivers = Vec::new(); + for (i, index) in receiver_index.iter_mut().enumerate() { + more_receivers |= Self::pull_receivers_for_shard(i as u8, index, &mut receivers)?; + } + Ok((more_receivers, receivers)) + } + + /// Pulls receiver data from the shard at `shard_index` starting at the `receiver_index`, + /// pushing the results back to `receivers`. #[inline] - fn ledger() -> Ledger { - Ledger(PhantomData) + fn pull_receivers_for_shard( + shard_index: u8, + receiver_index: &mut usize, + receivers: &mut ReceiverChunk, + ) -> Result { + let mut iter = if *receiver_index == 0 { + Shards::::iter_prefix(shard_index) + } else { + let raw_key = Shards::::hashed_key_for(shard_index, *receiver_index as u64 - 1); + Shards::::iter_prefix_from(shard_index, raw_key) + }; + for _ in 0..Self::PULL_MAX_PER_SHARD_UPDATE_SIZE { + match iter.next() { + Some((_, (utxo, encrypted_note))) => { + *receiver_index += 1; + receivers.push((decode(utxo)?, encrypted_note.try_into()?)); + } + _ => return Ok(false), + } + } + Ok(iter.next().is_some()) + } + + /// Pulls sender data from the ledger starting at the `sender_index`. + #[inline] + fn pull_senders( + sender_index: &mut usize, + ) -> Result<(bool, SenderChunk), scale_codec::Error> { + let mut senders = Vec::new(); + let mut iter = VoidNumberSet::::iter().skip(*sender_index); + for _ in 0..Self::PULL_MAX_SENDER_UPDATE_SIZE { + match iter.next() { + Some((sender, _)) => { + *sender_index += 1; + senders.push(decode(sender)?); + } + _ => return Ok((false, senders)), + } + } + Ok((iter.next().is_some(), senders)) } - /// The account ID of AssetManager + /// Returns the update required to be synchronized with the ledger starting from + /// `checkpoint`. + #[inline] + pub fn pull(mut checkpoint: Checkpoint) -> Result { + let (more_receivers, receivers) = Self::pull_receivers(&mut checkpoint.receiver_index)?; + let (more_senders, senders) = Self::pull_senders(&mut checkpoint.sender_index)?; + Ok(PullResponse { + should_continue: more_receivers || more_senders, + checkpoint, + receivers, + senders, + }) + } + + /// Returns the account ID of this pallet. + #[inline] pub fn account_id() -> T::AccountId { T::PalletId::get().into_account() } + + /// Posts the transaction encoded in `post` to the ledger, using `sources` and `sinks` as + /// the public deposit and public withdraw accounts respectively. + #[inline] + fn post_transaction( + origin: Option, + sources: Vec, + sinks: Vec, + post: TransferPost, + ) -> DispatchResultWithPostInfo { + Self::deposit_event( + config::TransferPost::try_from(post) + .map_err(|_| Error::::InvalidSerializedForm)? + .post(sources, sinks, &(), &mut Ledger(PhantomData)) + .map_err(Error::::from)? + .convert(origin), + ); + Ok(().into()) + } } } +/// Receiver Chunk Data +pub type ReceiverChunk = Vec<(config::Utxo, config::EncryptedNote)>; + +/// Sender Chunk Data +pub type SenderChunk = Vec; + +/// Pull Response +#[derive(Clone, Debug, Default, Eq, Hash, PartialEq)] +pub struct PullResponse { + /// Pull Continuation Flag + /// + /// The `should_continue` flag is set to `true` if the client should request more data from the + /// ledger to finish the pull. + pub should_continue: bool, + + /// Ledger Checkpoint + /// + /// If the `should_continue` flag is set to `true` then `checkpoint` is the next + /// [`Checkpoint`](Connection::Checkpoint) to request data from the ledger. Otherwise, it + /// represents the current ledger state. + pub checkpoint: Checkpoint, + + /// Ledger Receiver Chunk + pub receivers: ReceiverChunk, + + /// Ledger Sender Chunk + pub senders: SenderChunk, +} + /// Preprocessed Event enum PreprocessedEvent where @@ -600,16 +686,8 @@ where I: IntoIterator, { let _ = super_key; - let index = VoidNumberSetSize::::get(); - let mut i = 0; for (_, void_number) in iter { - let void_number = encode(&void_number.0); - VoidNumberSet::::insert(void_number, ()); - VoidNumberSetInsertionOrder::::insert(index + i, void_number); - i += 1; - } - if i != 0 { - VoidNumberSetSize::::set(index + i); + VoidNumberSet::::insert(encode(&void_number.0), ()); } } } diff --git a/pallets/manta-pay/src/types.rs b/pallets/manta-pay/src/types.rs index 4f9976c97..479d7f352 100644 --- a/pallets/manta-pay/src/types.rs +++ b/pallets/manta-pay/src/types.rs @@ -18,6 +18,7 @@ use super::*; use manta_util::into_array_unchecked; +use scale_codec::Error; /// Encodes the SCALE encodable `value` into a byte array with the given length `N`. #[inline] @@ -31,11 +32,11 @@ where /// Decodes the `bytes` array of the given length `N` into the SCALE decodable type `T` returning a /// blanket error if decoding fails. #[inline] -pub(crate) fn decode(bytes: [u8; N]) -> Result +pub(crate) fn decode(bytes: [u8; N]) -> Result where T: Decode, { - T::decode(&mut bytes.as_slice()).map_err(|_| ()) + T::decode(&mut bytes.as_slice()) } /// Asset @@ -101,7 +102,7 @@ impl From for EncryptedNote { } impl TryFrom for config::EncryptedNote { - type Error = (); + type Error = Error; #[inline] fn try_from(encrypted_note: EncryptedNote) -> Result { @@ -133,7 +134,7 @@ impl From for SenderPost { } impl TryFrom for config::SenderPost { - type Error = (); + type Error = Error; #[inline] fn try_from(post: SenderPost) -> Result { @@ -165,7 +166,7 @@ impl From for ReceiverPost { } impl TryFrom for config::ReceiverPost { - type Error = (); + type Error = Error; #[inline] fn try_from(post: ReceiverPost) -> Result { @@ -213,7 +214,7 @@ impl From for TransferPost { } impl TryFrom for config::TransferPost { - type Error = (); + type Error = Error; #[inline] fn try_from(post: TransferPost) -> Result {