From 6c4576dc4376c2a1541366fe873d926cff29f8cc Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Mon, 26 Aug 2024 15:21:04 +0100 Subject: [PATCH 1/5] created link nullifiers fn and DRYed scan results --- zingo-sync/src/primitives.rs | 2 +- zingo-sync/src/sync.rs | 119 ++++++++++++++++++++++++++++------- zingo-sync/src/traits.rs | 5 +- 3 files changed, 100 insertions(+), 26 deletions(-) diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs index 26a581899d..458ee559fa 100644 --- a/zingo-sync/src/primitives.rs +++ b/zingo-sync/src/primitives.rs @@ -56,7 +56,7 @@ impl OutputId { } /// Binary tree map of nullifiers from transaction spends or actions -#[derive(Debug, MutGetters)] +#[derive(Debug, Getters, MutGetters)] #[getset(get = "pub", get_mut = "pub")] pub struct NullifierMap { sapling: BTreeMap, diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs index 0a452cbadb..cfe16370ec 100644 --- a/zingo-sync/src/sync.rs +++ b/zingo-sync/src/sync.rs @@ -1,6 +1,7 @@ //! Entrypoint for sync engine use std::cmp; +use std::collections::HashSet; use std::ops::Range; use std::time::Duration; @@ -26,7 +27,7 @@ const BATCH_SIZE: u32 = 10; /// Syncs a wallet to the latest state of the blockchain pub async fn sync( - client: CompactTxStreamerClient, + client: CompactTxStreamerClient, // TODO: change underlying service for generic parameters: &P, wallet: &mut W, ) -> Result<(), ()> @@ -65,7 +66,7 @@ where let mut interval = tokio::time::interval(Duration::from_millis(30)); loop { - interval.tick().await; + interval.tick().await; // TODO: tokio select to recieve scan results before tick // if a scan worker is idle, send it a new scan task if scanner.is_worker_idle() { @@ -86,10 +87,7 @@ where match scan_results_receiver.try_recv() { Ok((scan_range, scan_results)) => { - update_wallet_data(wallet, scan_results).unwrap(); - // TODO: link nullifiers and scan linked transactions - remove_irrelevant_data(wallet, &scan_range).unwrap(); - mark_scanned(scan_range, wallet.get_sync_state_mut().unwrap()).unwrap(); + process_scan_results(wallet, scan_range, scan_results).unwrap() } Err(TryRecvError::Empty) => (), Err(TryRecvError::Disconnected) => break, @@ -98,10 +96,7 @@ where drop(scanner); while let Some((scan_range, scan_results)) = scan_results_receiver.recv().await { - update_wallet_data(wallet, scan_results).unwrap(); - // TODO: link nullifiers and scan linked transactions - remove_irrelevant_data(wallet, &scan_range).unwrap(); - mark_scanned(scan_range, wallet.get_sync_state_mut().unwrap()).unwrap(); + process_scan_results(wallet, scan_range, scan_results).unwrap(); } try_join_all(handles).await.unwrap(); @@ -159,6 +154,7 @@ where // TODO: add logic to combine chain tip scan range with wallet tip scan range // TODO: add scan priority logic + // TODO: replace `ignored` (a.k.a scanning) priority with `verify` to prioritise ranges that were being scanned when sync was interrupted Ok(()) } @@ -193,20 +189,18 @@ fn prepare_next_scan_range(sync_state: &mut SyncState) -> Option { } } -fn mark_scanned(scan_range: ScanRange, sync_state: &mut SyncState) -> Result<(), ()> { - let scan_ranges = sync_state.scan_ranges_mut(); - - if let Some((index, range)) = scan_ranges - .iter() - .enumerate() - .find(|(_, range)| range.block_range() == scan_range.block_range()) - { - scan_ranges[index] = - ScanRange::from_parts(range.block_range().clone(), ScanPriority::Scanned); - } else { - panic!("scanned range not found!") - } - // TODO: also combine adjacent scanned ranges together +fn process_scan_results( + wallet: &mut W, + scan_range: ScanRange, + scan_results: ScanResults, +) -> Result<(), ()> +where + W: SyncWallet + SyncBlocks + SyncTransactions + SyncNullifiers + SyncShardTrees, +{ + update_wallet_data(wallet, scan_results).unwrap(); + link_nullifiers(wallet).unwrap(); + remove_irrelevant_data(wallet, &scan_range).unwrap(); + mark_scanned(scan_range, wallet.get_sync_state_mut().unwrap()).unwrap(); Ok(()) } @@ -234,6 +228,65 @@ where Ok(()) } +fn link_nullifiers(wallet: &mut W) -> Result<(), ()> +where + W: SyncWallet + SyncTransactions + SyncNullifiers, +{ + let wallet_transactions = wallet.get_wallet_transactions().unwrap(); + let wallet_txids = wallet_transactions.keys().copied().collect::>(); + let sapling_nullifiers = wallet_transactions + .values() + .flat_map(|tx| tx.sapling_notes()) + .flat_map(|note| note.nullifier()) + .collect::>(); + let orchard_nullifiers = wallet_transactions + .values() + .flat_map(|tx| tx.orchard_notes()) + .flat_map(|note| note.nullifier()) + .collect::>(); + + let mut spend_locations = Vec::new(); + let nullifier_map = wallet.get_nullifiers_mut().unwrap(); + spend_locations.extend( + sapling_nullifiers + .iter() + .flat_map(|nf| nullifier_map.sapling_mut().remove(&nf)), + ); + spend_locations.extend( + orchard_nullifiers + .iter() + .flat_map(|nf| nullifier_map.orchard_mut().remove(&nf)), + ); + + for (block_height, txid) in spend_locations { + // skip found spend if transaction already exists in the wallet + if wallet_txids.get(&txid).is_some() { + continue; + } + + let scan_ranges = wallet.get_sync_state_mut().unwrap().scan_ranges_mut(); + let (range_index, scan_range) = scan_ranges + .iter() + .enumerate() + .find(|(_, range)| range.block_range().contains(&block_height)) + .expect("scan range should always exist for mapped nullifiers"); + + // if the scan range with the found spend is already scanned, the wallet blocks will already be stored and the transaction can be scanned + // if the scan range is currently being scanned (has `Ignored` priority), TODO: explain how the txid is stored for future scanning + // otherwise, create a scan range with `FoundNote` priority TODO: explain how txid is stored here also + if scan_range.priority() == ScanPriority::Scanned { + // TODO: scan tx + } else if scan_range.priority() == ScanPriority::Ignored { + // TODO: store txid for scanning + } else { + // TODO: store txid for scanning + // TODO: create found note range + } + } + + Ok(()) +} + fn remove_irrelevant_data(wallet: &mut W, scan_range: &ScanRange) -> Result<(), ()> where W: SyncWallet + SyncBlocks + SyncNullifiers + SyncTransactions, @@ -275,3 +328,21 @@ where Ok(()) } + +fn mark_scanned(scan_range: ScanRange, sync_state: &mut SyncState) -> Result<(), ()> { + let scan_ranges = sync_state.scan_ranges_mut(); + + if let Some((index, range)) = scan_ranges + .iter() + .enumerate() + .find(|(_, range)| range.block_range() == scan_range.block_range()) + { + scan_ranges[index] = + ScanRange::from_parts(range.block_range().clone(), ScanPriority::Scanned); + } else { + panic!("scanned range not found!") + } + // TODO: also combine adjacent scanned ranges together + + Ok(()) +} diff --git a/zingo-sync/src/traits.rs b/zingo-sync/src/traits.rs index 9fbbceb4c6..d339655fe0 100644 --- a/zingo-sync/src/traits.rs +++ b/zingo-sync/src/traits.rs @@ -57,7 +57,7 @@ pub trait SyncBlocks: SyncWallet { /// Trait for interfacing [`crate::primitives::WalletTransaction`]s with wallet data pub trait SyncTransactions: SyncWallet { - /// Get mutable reference to wallet transactions + /// Get wallet transactions fn get_wallet_transactions(&self) -> Result<&HashMap, Self::Error>; /// Get mutable reference to wallet transactions @@ -81,6 +81,9 @@ pub trait SyncTransactions: SyncWallet { pub trait SyncNullifiers: SyncWallet { // TODO: add method to get wallet data for writing defualt implementations on other methods + // /// Get wallet nullifier map + // fn get_nullifiers(&self) -> Result<&NullifierMap, Self::Error>; + /// Get mutable reference to wallet nullifier map fn get_nullifiers_mut(&mut self) -> Result<&mut NullifierMap, Self::Error>; From fbd1b0e97f6a9914c1e2d40eeb5c7866aa872472 Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Tue, 27 Aug 2024 16:11:12 +0100 Subject: [PATCH 2/5] completed nullifier linking --- zingo-sync/src/primitives.rs | 5 ++ zingo-sync/src/sync.rs | 115 ++++++++++++++++++++++++++++------- 2 files changed, 98 insertions(+), 22 deletions(-) diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs index 458ee559fa..2ed0c26e75 100644 --- a/zingo-sync/src/primitives.rs +++ b/zingo-sync/src/primitives.rs @@ -20,7 +20,11 @@ use crate::{keys::KeyId, utils}; #[derive(Debug, Getters, MutGetters)] #[getset(get = "pub", get_mut = "pub")] pub struct SyncState { + /// A vec of block ranges with scan priorities from wallet birthday to chain tip. + /// In block height order with no overlaps or gaps. scan_ranges: Vec, + /// Block height and txid of known spends which are awaiting the scanning of the range it belongs to for transaction decryption. + spend_locations: Vec<(BlockHeight, TxId)>, } impl SyncState { @@ -28,6 +32,7 @@ impl SyncState { pub fn new() -> Self { SyncState { scan_ranges: Vec::new(), + spend_locations: Vec::new(), } } } diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs index cfe16370ec..ff209e4839 100644 --- a/zingo-sync/src/sync.rs +++ b/zingo-sync/src/sync.rs @@ -22,6 +22,7 @@ use zcash_primitives::consensus::{BlockHeight, NetworkUpgrade, Parameters}; use futures::future::try_join_all; use tokio::sync::mpsc; +// TODO; replace fixed batches with orchard shard ranges (block ranges containing all note commitments to an orchard shard or fragment of a shard) const BATCH_SIZE: u32 = 10; // const BATCH_SIZE: u32 = 1_000; @@ -250,43 +251,113 @@ where spend_locations.extend( sapling_nullifiers .iter() - .flat_map(|nf| nullifier_map.sapling_mut().remove(&nf)), + .flat_map(|nf| nullifier_map.sapling_mut().remove(nf)), ); spend_locations.extend( orchard_nullifiers .iter() - .flat_map(|nf| nullifier_map.orchard_mut().remove(&nf)), + .flat_map(|nf| nullifier_map.orchard_mut().remove(nf)), ); + let sync_state = wallet.get_sync_state_mut().unwrap(); for (block_height, txid) in spend_locations { - // skip found spend if transaction already exists in the wallet - if wallet_txids.get(&txid).is_some() { + // skip if transaction already exists in the wallet + if wallet_txids.contains(&txid) { continue; } - let scan_ranges = wallet.get_sync_state_mut().unwrap().scan_ranges_mut(); - let (range_index, scan_range) = scan_ranges - .iter() - .enumerate() - .find(|(_, range)| range.block_range().contains(&block_height)) - .expect("scan range should always exist for mapped nullifiers"); - - // if the scan range with the found spend is already scanned, the wallet blocks will already be stored and the transaction can be scanned - // if the scan range is currently being scanned (has `Ignored` priority), TODO: explain how the txid is stored for future scanning - // otherwise, create a scan range with `FoundNote` priority TODO: explain how txid is stored here also - if scan_range.priority() == ScanPriority::Scanned { - // TODO: scan tx - } else if scan_range.priority() == ScanPriority::Ignored { - // TODO: store txid for scanning - } else { - // TODO: store txid for scanning - // TODO: create found note range - } + sync_state.spend_locations_mut().push((block_height, txid)); + update_scan_priority(sync_state, block_height, ScanPriority::FoundNote); } Ok(()) } +/// Splits out a scan range surrounding a given block height with the specified priority +fn update_scan_priority( + sync_state: &mut SyncState, + block_height: BlockHeight, + scan_priority: ScanPriority, +) { + let (index, scan_range) = sync_state + .scan_ranges() + .iter() + .enumerate() + .find(|(_, range)| range.block_range().contains(&block_height)) + .expect("scan range should always exist for mapped nullifiers"); + + // Skip if the given block height is within a range that is scanned or being scanning + if scan_range.priority() == ScanPriority::Scanned + || scan_range.priority() == ScanPriority::Ignored + { + return; + } + + let new_block_range = determine_block_range(block_height); + let split_ranges = split_out_scan_range(scan_range, new_block_range, scan_priority); + sync_state + .scan_ranges_mut() + .splice(index..=index, split_ranges); +} + +/// Determines which range of blocks should be scanned for a given block height +fn determine_block_range(block_height: BlockHeight) -> Range { + let start = block_height - (u32::from(block_height) % BATCH_SIZE); // TODO: will be replaced with first block of associated orchard shard + let end = start + BATCH_SIZE; // TODO: will be replaced with last block of associated orchard shard + Range { start, end } +} + +/// Takes a scan range and splits it at [block_range.start] and [block_range.end], returning a vec of scan ranges where +/// the scan range with the specified [block_range] has the given [scan_priority]. +/// +/// If [block_range] goes beyond the bounds of [scan_range.block_range()] no splitting will occur at the upper and/or +/// lower bound but the priority will still be updated +/// +/// Panics if no blocks in [block_range] are contained within [scan_range.block_range()] +fn split_out_scan_range( + scan_range: &ScanRange, + block_range: Range, + scan_priority: ScanPriority, +) -> Vec { + let mut split_ranges = Vec::new(); + if let Some((lower_range, higher_range)) = scan_range.split_at(block_range.start) { + split_ranges.push(lower_range); + if let Some((middle_range, higher_range)) = higher_range.split_at(block_range.end) { + // [scan_range] is split at the upper and lower bound of [block_range] + split_ranges.push(ScanRange::from_parts( + middle_range.block_range().clone(), + scan_priority, + )); + split_ranges.push(higher_range); + } else { + // [scan_range] is split only at the lower bound of [block_range] + split_ranges.push(ScanRange::from_parts( + higher_range.block_range().clone(), + scan_priority, + )); + } + } else if let Some((lower_range, higher_range)) = scan_range.split_at(block_range.end) { + // [scan_range] is split only at the upper bound of [block_range] + split_ranges.push(ScanRange::from_parts( + lower_range.block_range().clone(), + scan_priority, + )); + split_ranges.push(higher_range); + } else { + // [scan_range] is not split as it is fully contained within [block_range] + // only scan priority is updated + assert!(scan_range.block_range().start >= block_range.start); + assert!(scan_range.block_range().end <= block_range.end); + + split_ranges.push(ScanRange::from_parts( + scan_range.block_range().clone(), + scan_priority, + )); + }; + + split_ranges +} + fn remove_irrelevant_data(wallet: &mut W, scan_range: &ScanRange) -> Result<(), ()> where W: SyncWallet + SyncBlocks + SyncNullifiers + SyncTransactions, From 0778f5970ddc60dde3c35fc0e6b36b9f07169f18 Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Wed, 28 Aug 2024 10:11:04 +0100 Subject: [PATCH 3/5] scan spending transactions with no change --- zingo-sync/src/scan.rs | 6 +-- zingo-sync/src/sync.rs | 112 ++++++++++++++++++++++++++++++++--------- 2 files changed, 90 insertions(+), 28 deletions(-) diff --git a/zingo-sync/src/scan.rs b/zingo-sync/src/scan.rs index 87c39e8637..8f342806a0 100644 --- a/zingo-sync/src/scan.rs +++ b/zingo-sync/src/scan.rs @@ -22,7 +22,7 @@ use self::{compact_blocks::scan_compact_blocks, transactions::scan_transactions} mod compact_blocks; pub(crate) mod task; -mod transactions; +pub(crate) mod transactions; struct InitialScanData { previous_block: Option, @@ -120,13 +120,13 @@ pub(crate) struct ScanResults { pub(crate) shard_tree_data: ShardTreeData, } -struct DecryptedNoteData { +pub(crate) struct DecryptedNoteData { sapling_nullifiers_and_positions: HashMap, orchard_nullifiers_and_positions: HashMap, } impl DecryptedNoteData { - fn new() -> Self { + pub(crate) fn new() -> Self { DecryptedNoteData { sapling_nullifiers_and_positions: HashMap::new(), orchard_nullifiers_and_positions: HashMap::new(), diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs index ff209e4839..3626bd457d 100644 --- a/zingo-sync/src/sync.rs +++ b/zingo-sync/src/sync.rs @@ -1,15 +1,17 @@ //! Entrypoint for sync engine use std::cmp; -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::ops::Range; use std::time::Duration; -use crate::client::FetchRequest; -use crate::client::{fetch::fetch, get_chain_height}; +use crate::client::fetch::fetch; +use crate::client::{self, FetchRequest}; +use crate::keys::ScanningKeys; use crate::primitives::SyncState; use crate::scan::task::{ScanTask, Scanner}; -use crate::scan::ScanResults; +use crate::scan::transactions::scan_transactions; +use crate::scan::{DecryptedNoteData, ScanResults}; use crate::traits::{SyncBlocks, SyncNullifiers, SyncShardTrees, SyncTransactions, SyncWallet}; use tokio::sync::mpsc::error::TryRecvError; @@ -59,12 +61,15 @@ where let ufvks = wallet.get_unified_full_viewing_keys().unwrap(); let mut scanner = Scanner::new( scan_results_sender, - fetch_request_sender, + fetch_request_sender.clone(), parameters.clone(), - ufvks, + ufvks.clone(), ); scanner.spawn_workers(); + // TODO: replace scanning keys with ufvk for scan_tx + let scanning_keys = ScanningKeys::from_account_ufvks(ufvks); + let mut interval = tokio::time::interval(Duration::from_millis(30)); loop { interval.tick().await; // TODO: tokio select to recieve scan results before tick @@ -87,9 +92,16 @@ where } match scan_results_receiver.try_recv() { - Ok((scan_range, scan_results)) => { - process_scan_results(wallet, scan_range, scan_results).unwrap() - } + Ok((scan_range, scan_results)) => process_scan_results( + wallet, + fetch_request_sender.clone(), + parameters, + &scanning_keys, + scan_range, + scan_results, + ) + .await + .unwrap(), Err(TryRecvError::Empty) => (), Err(TryRecvError::Disconnected) => break, } @@ -97,15 +109,25 @@ where drop(scanner); while let Some((scan_range, scan_results)) = scan_results_receiver.recv().await { - process_scan_results(wallet, scan_range, scan_results).unwrap(); + process_scan_results( + wallet, + fetch_request_sender.clone(), + parameters, + &scanning_keys, + scan_range, + scan_results, + ) + .await + .unwrap(); } + drop(fetch_request_sender); try_join_all(handles).await.unwrap(); Ok(()) } -// update scan_ranges to include blocks between the last known chain height (wallet height) and the chain height from the server +/// Update scan ranges to include blocks between the last known chain height (wallet height) and the chain height from the server async fn update_scan_ranges

( fetch_request_sender: mpsc::UnboundedSender, parameters: &P, @@ -115,7 +137,9 @@ async fn update_scan_ranges

( where P: Parameters, { - let chain_height = get_chain_height(fetch_request_sender).await.unwrap(); + let chain_height = client::get_chain_height(fetch_request_sender) + .await + .unwrap(); let scan_ranges = sync_state.scan_ranges_mut(); @@ -160,7 +184,8 @@ where Ok(()) } -// returns `None` if there are no more ranges to scan +/// Prepares the next scan range for scanning. +/// Returns `None` if there are no more ranges to scan fn prepare_next_scan_range(sync_state: &mut SyncState) -> Option { let scan_ranges = sync_state.scan_ranges_mut(); @@ -190,16 +215,23 @@ fn prepare_next_scan_range(sync_state: &mut SyncState) -> Option { } } -fn process_scan_results( +/// Scan post-processing +async fn process_scan_results( wallet: &mut W, + fetch_request_sender: mpsc::UnboundedSender, + parameters: &P, + scanning_keys: &ScanningKeys, scan_range: ScanRange, scan_results: ScanResults, ) -> Result<(), ()> where + P: Parameters, W: SyncWallet + SyncBlocks + SyncTransactions + SyncNullifiers + SyncShardTrees, { update_wallet_data(wallet, scan_results).unwrap(); - link_nullifiers(wallet).unwrap(); + link_nullifiers(wallet, fetch_request_sender, parameters, scanning_keys) + .await + .unwrap(); remove_irrelevant_data(wallet, &scan_range).unwrap(); mark_scanned(scan_range, wallet.get_sync_state_mut().unwrap()).unwrap(); @@ -229,9 +261,15 @@ where Ok(()) } -fn link_nullifiers(wallet: &mut W) -> Result<(), ()> +async fn link_nullifiers( + wallet: &mut W, + fetch_request_sender: mpsc::UnboundedSender, + parameters: &P, + scanning_keys: &ScanningKeys, +) -> Result<(), ()> where - W: SyncWallet + SyncTransactions + SyncNullifiers, + P: Parameters, + W: SyncBlocks + SyncTransactions + SyncNullifiers, { let wallet_transactions = wallet.get_wallet_transactions().unwrap(); let wallet_txids = wallet_transactions.keys().copied().collect::>(); @@ -246,34 +284,58 @@ where .flat_map(|note| note.nullifier()) .collect::>(); - let mut spend_locations = Vec::new(); + let mut spend_locators = Vec::new(); let nullifier_map = wallet.get_nullifiers_mut().unwrap(); - spend_locations.extend( + spend_locators.extend( sapling_nullifiers .iter() .flat_map(|nf| nullifier_map.sapling_mut().remove(nf)), ); - spend_locations.extend( + spend_locators.extend( orchard_nullifiers .iter() .flat_map(|nf| nullifier_map.orchard_mut().remove(nf)), ); - let sync_state = wallet.get_sync_state_mut().unwrap(); - for (block_height, txid) in spend_locations { + // in the edge case where a spending transaction received no change, scan the transactions that evaded trial decryption + let mut spending_txids = HashSet::new(); + let mut wallet_blocks = BTreeMap::new(); + for (block_height, txid) in spend_locators.iter() { // skip if transaction already exists in the wallet - if wallet_txids.contains(&txid) { + if wallet_txids.contains(txid) { continue; } - sync_state.spend_locations_mut().push((block_height, txid)); - update_scan_priority(sync_state, block_height, ScanPriority::FoundNote); + spending_txids.insert(*txid); + wallet_blocks.insert( + *block_height, + wallet.get_wallet_block(*block_height).unwrap(), + ); + } + let spending_transactions = scan_transactions( + fetch_request_sender, + parameters, + scanning_keys, + spending_txids, + DecryptedNoteData::new(), + &wallet_blocks, + ) + .await + .unwrap(); + wallet + .extend_wallet_transactions(spending_transactions) + .unwrap(); + + if !spend_locators.is_empty() { + + // TODO: add spent field to output and change to Some(TxId) } Ok(()) } /// Splits out a scan range surrounding a given block height with the specified priority +#[allow(dead_code)] fn update_scan_priority( sync_state: &mut SyncState, block_height: BlockHeight, From 51850cb680dad46006eb7953e96755caacf66f58 Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Wed, 28 Aug 2024 12:14:55 +0100 Subject: [PATCH 4/5] add spending transaction to spent notes --- zingo-sync/src/primitives.rs | 31 ++++++++++++------ zingo-sync/src/scan/transactions.rs | 7 ++-- zingo-sync/src/sync.rs | 50 ++++++++++++++++++++++------- 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs index 2ed0c26e75..aad354242a 100644 --- a/zingo-sync/src/primitives.rs +++ b/zingo-sync/src/primitives.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; -use getset::{CopyGetters, Getters, MutGetters}; +use getset::{CopyGetters, Getters, MutGetters, Setters}; use incrementalmerkletree::Position; use zcash_client_backend::data_api::scanning::ScanRange; @@ -124,11 +124,10 @@ impl WalletBlock { } /// Wallet transaction -#[derive(Debug, CopyGetters)] -#[getset(get_copy = "pub")] +#[derive(Debug, Getters, CopyGetters)] pub struct WalletTransaction { - #[getset(get_copy = "pub")] - txid: TxId, + #[getset(get = "pub")] + transaction: zcash_primitives::transaction::Transaction, #[getset(get_copy = "pub")] block_height: BlockHeight, #[getset(skip)] @@ -143,7 +142,7 @@ pub struct WalletTransaction { impl WalletTransaction { pub fn from_parts( - txid: TxId, + transaction: zcash_primitives::transaction::Transaction, block_height: BlockHeight, sapling_notes: Vec, orchard_notes: Vec, @@ -151,7 +150,7 @@ impl WalletTransaction { outgoing_orchard_notes: Vec, ) -> Self { Self { - txid, + transaction, block_height, sapling_notes, orchard_notes, @@ -164,10 +163,18 @@ impl WalletTransaction { &self.sapling_notes } + pub fn sapling_notes_mut(&mut self) -> Vec<&mut SaplingNote> { + self.sapling_notes.iter_mut().collect() + } + pub fn orchard_notes(&self) -> &[OrchardNote] { &self.orchard_notes } + pub fn orchard_notes_mut(&mut self) -> Vec<&mut OrchardNote> { + self.orchard_notes.iter_mut().collect() + } + pub fn outgoing_sapling_notes(&self) -> &[OutgoingSaplingNote] { &self.outgoing_sapling_notes } @@ -181,7 +188,7 @@ pub type SaplingNote = WalletNote; /// Wallet note, shielded output with metadata relevant to the wallet -#[derive(Debug, Getters, CopyGetters)] +#[derive(Debug, Getters, CopyGetters, Setters)] pub struct WalletNote { /// Output ID #[getset(get_copy = "pub")] @@ -201,6 +208,8 @@ pub struct WalletNote { /// Memo #[getset(get = "pub")] memo: Memo, + #[getset(get = "pub", set = "pub")] + spending_transaction: Option, } impl WalletNote { @@ -211,6 +220,7 @@ impl WalletNote { nullifier: Option, position: Position, memo: Memo, + spending_transaction: Option, ) -> Self { Self { output_id, @@ -219,6 +229,7 @@ impl WalletNote { nullifier, position, memo, + spending_transaction, } } } @@ -227,7 +238,7 @@ pub type OutgoingSaplingNote = OutgoingNote; pub type OutgoingOrchardNote = OutgoingNote; /// Note sent from this capability to a recipient -#[derive(Debug, Clone, Getters, CopyGetters, MutGetters)] +#[derive(Debug, Clone, Getters, CopyGetters, Setters)] pub struct OutgoingNote { /// Output ID #[getset(get_copy = "pub")] @@ -242,7 +253,7 @@ pub struct OutgoingNote { #[getset(get = "pub")] memo: Memo, /// Recipient's full unified address from encoded memo - #[getset(get = "pub", get_mut = "pub")] + #[getset(get = "pub", set = "pub")] recipient_ua: Option, } diff --git a/zingo-sync/src/scan/transactions.rs b/zingo-sync/src/scan/transactions.rs index aa1dccbb62..60f6ff8800 100644 --- a/zingo-sync/src/scan/transactions.rs +++ b/zingo-sync/src/scan/transactions.rs @@ -219,7 +219,7 @@ fn scan_transaction( } } Ok(WalletTransaction::from_parts( - transaction.txid(), + transaction, block_height, sapling_notes, orchard_notes, @@ -257,6 +257,7 @@ where Some(*nullifier), *position, Memo::from_bytes(memo_bytes.as_ref()).unwrap(), + None, )); } } @@ -359,6 +360,8 @@ fn add_recipient_unified_address( outgoing_notes .iter_mut() .filter(|note| ua_receivers.contains(¬e.encoded_recipient(parameters))) - .for_each(|note| *note.recipient_ua_mut() = Some(ua.clone())) + .for_each(|note| { + note.set_recipient_ua(Some(ua.clone())); + }); } } diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs index 3626bd457d..6512f47b49 100644 --- a/zingo-sync/src/sync.rs +++ b/zingo-sync/src/sync.rs @@ -23,6 +23,7 @@ use zcash_primitives::consensus::{BlockHeight, NetworkUpgrade, Parameters}; use futures::future::try_join_all; use tokio::sync::mpsc; +use zcash_primitives::transaction::TxId; // TODO; replace fixed batches with orchard shard ranges (block ranges containing all note commitments to an orchard shard or fragment of a shard) const BATCH_SIZE: u32 = 10; @@ -271,6 +272,7 @@ where P: Parameters, W: SyncBlocks + SyncTransactions + SyncNullifiers, { + // locate spends let wallet_transactions = wallet.get_wallet_transactions().unwrap(); let wallet_txids = wallet_transactions.keys().copied().collect::>(); let sapling_nullifiers = wallet_transactions @@ -284,23 +286,25 @@ where .flat_map(|note| note.nullifier()) .collect::>(); - let mut spend_locators = Vec::new(); let nullifier_map = wallet.get_nullifiers_mut().unwrap(); - spend_locators.extend( + let sapling_spend_locators: BTreeMap = sapling_nullifiers .iter() - .flat_map(|nf| nullifier_map.sapling_mut().remove(nf)), - ); - spend_locators.extend( + .flat_map(|nf| nullifier_map.sapling_mut().remove_entry(nf)) + .collect(); + let orchard_spend_locators: BTreeMap = orchard_nullifiers .iter() - .flat_map(|nf| nullifier_map.orchard_mut().remove(nf)), - ); + .flat_map(|nf| nullifier_map.orchard_mut().remove_entry(nf)) + .collect(); // in the edge case where a spending transaction received no change, scan the transactions that evaded trial decryption let mut spending_txids = HashSet::new(); let mut wallet_blocks = BTreeMap::new(); - for (block_height, txid) in spend_locators.iter() { + for (block_height, txid) in sapling_spend_locators + .values() + .chain(orchard_spend_locators.values()) + { // skip if transaction already exists in the wallet if wallet_txids.contains(txid) { continue; @@ -326,10 +330,32 @@ where .extend_wallet_transactions(spending_transactions) .unwrap(); - if !spend_locators.is_empty() { - - // TODO: add spent field to output and change to Some(TxId) - } + // add spending transaction for all spent notes + let wallet_transactions = wallet.get_wallet_transactions_mut().unwrap(); + wallet_transactions + .values_mut() + .flat_map(|tx| tx.sapling_notes_mut()) + .filter(|note| note.spending_transaction().is_none()) + .for_each(|note| { + if let Some((_, txid)) = note + .nullifier() + .and_then(|nf| sapling_spend_locators.get(&nf)) + { + note.set_spending_transaction(Some(*txid)); + } + }); + wallet_transactions + .values_mut() + .flat_map(|tx| tx.orchard_notes_mut()) + .filter(|note| note.spending_transaction().is_none()) + .for_each(|note| { + if let Some((_, txid)) = note + .nullifier() + .and_then(|nf| orchard_spend_locators.get(&nf)) + { + note.set_spending_transaction(Some(*txid)); + } + }); Ok(()) } From 7c8abdcc29b39331caf35f4b8903c0ab997fbb3b Mon Sep 17 00:00:00 2001 From: Za Wilcox Date: Fri, 15 Nov 2024 15:18:15 -0700 Subject: [PATCH 5/5] Update zingo-sync/src/sync.rs Co-authored-by: Hazel OHearn Signed-off-by: Za Wilcox --- zingo-sync/src/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs index 752781a971..a0131258a3 100644 --- a/zingo-sync/src/sync.rs +++ b/zingo-sync/src/sync.rs @@ -73,7 +73,7 @@ where let mut interval = tokio::time::interval(Duration::from_millis(30)); loop { - interval.tick().await; // TODO: tokio select to recieve scan results before tick + interval.tick().await; // TODO: tokio select to receive scan results before tick // if a scan worker is idle, send it a new scan task if scanner.is_worker_idle() {