From 60399c65360ee97ab0cf77a8ab521759f9374984 Mon Sep 17 00:00:00 2001 From: Alessandro Rezzi Date: Wed, 20 Nov 2024 15:37:39 +0100 Subject: [PATCH] Shield Activity (#416) * During the initial sync, add to the wallet txs that are decrypted by the Shield manager * Make getCredit and getDebit return whether all vins/vouts belongs to the wallet * Update HistoricalTx class, to take in account shield data * make toHistoricalTXs take in account shield data * make the Activity show shielded txs * Update broken tests * Add missing await * Make old wallet resync if load operation is not succesful * Add utils function to reverse and swap endianess * Simplify logic and fix bug of sent to shield address * Remove unused imports * DebugLog in case of failure * apply new lint rules * fix: load first the shield from disk * fix: Do not use txSelfMap for DELEGATION * feat: Use new pivx-shield beta version * beautify numbers * Update E2E tests (new icons) * remove merge trash * fix infinite await bug * fix: Use block.time instead of mediantime * handle block at batches of 50 --- cypress/snapshots/html/snapshot.html | 2 +- scripts/dashboard/Activity.vue | 102 +++++++++++++------- scripts/historical_tx.js | 17 +++- scripts/mempool.js | 38 +++++--- scripts/reader.js | 4 +- scripts/utils.js | 4 + scripts/wallet.js | 139 +++++++++++++++++++-------- tests/unit/mempool.spec.js | 12 +-- 8 files changed, 214 insertions(+), 104 deletions(-) diff --git a/cypress/snapshots/html/snapshot.html b/cypress/snapshots/html/snapshot.html index 77c9b9700..30dad255d 100644 --- a/cypress/snapshots/html/snapshot.html +++ b/cypress/snapshots/html/snapshot.html @@ -134,7 +134,7 @@ -Sep 30Sent to self0.01 PIV +Sep 30Sent to self0.01 PIV diff --git a/scripts/dashboard/Activity.vue b/scripts/dashboard/Activity.vue index d62cab7a5..fc3fdf2bc 100644 --- a/scripts/dashboard/Activity.vue +++ b/scripts/dashboard/Activity.vue @@ -8,11 +8,11 @@ import { Database } from '../database.js'; import { HistoricalTx, HistoricalTxType } from '../historical_tx.js'; import { getNameOrAddress } from '../contacts-book.js'; import { getEventEmitter } from '../event_bus'; -import { beautifyNumber } from '../misc'; import iCheck from '../../assets/icons/icon-check.svg'; import iHourglass from '../../assets/icons/icon-hourglass.svg'; import { blockCount } from '../global.js'; +import { beautifyNumber } from '../misc.js'; const props = defineProps({ title: String, @@ -69,6 +69,39 @@ const txMap = computed(() => { }; }); +/** + * Returns the information that we need to show (icon + label + amount) for a self transaction + * @param {number} amount - The net amount of transparent PIVs in a transaction + * @param {number} shieldAmount - The net amount of shielded PIVs in a transaction + */ +function txSelfMap(amount, shieldAmount) { + if (shieldAmount == 0 || amount == 0) { + return { + icon: 'fa-recycle', + colour: 'white', + content: + shieldAmount == 0 + ? translation.activitySentTo + : 'Shield sent to self', + amount: Math.abs(shieldAmount + amount), + }; + } else if (shieldAmount > 0) { + return { + icon: 'fa-shield', + colour: 'white', + content: 'Shielding', + amount: shieldAmount, + }; + } else if (shieldAmount < 0) { + return { + icon: 'fa-shield', + colour: 'white', + content: 'De-Shielding', + amount: amount, + }; + } +} + function updateReward() { if (!props.rewards) return; let res = 0; @@ -78,6 +111,7 @@ function updateReward() { } rewardAmount.value = res; } + async function update(txToAdd = 0) { // Return if wallet is not synced yet if (!wallet.isSynced) { @@ -168,6 +202,8 @@ async function parseTXs(arrTXs) { } } + let amountToShow = Math.abs(cTx.amount + cTx.shieldAmount); + // Coinbase Transactions (rewards) require coinbaseMaturity confs let fConfirmed = cTx.blockHeight > 0 && @@ -176,44 +212,20 @@ async function parseTXs(arrTXs) { ? cChainParams.current.coinbaseMaturity : 6); - // Format the amount to reduce text size - let formattedAmt = ''; - if (cTx.amount < 0.01) { - formattedAmt = beautifyNumber('0.01', '13px'); - } else if (cTx.amount >= 100) { - formattedAmt = beautifyNumber( - Math.round(cTx.amount).toString(), - '13px' - ); - } else { - formattedAmt = beautifyNumber(`${cTx.amount.toFixed(2)}`, '13px'); - } - - // For 'Send' TXs: Check if this is a send-to-self transaction - let fSendToSelf = false; - if (cTx.type === HistoricalTxType.SENT) { - fSendToSelf = true; - // Check all addresses to find our own, caching them for performance - for (const strAddr of cTx.receivers) { - // If a previous Tx checked this address, skip it, otherwise, check it against our own address(es) - if (!wallet.isOwnAddress(strAddr)) { - // External address, this is not a self-only Tx - fSendToSelf = false; - } - } - } - // Take the icon, colour and content based on the type of the transaction let { icon, colour, content } = txMap.value[cTx.type]; const match = content.match(/{(.)}/); if (match) { let who = ''; - if (fSendToSelf) { + if (cTx.isToSelf && cTx.type !== HistoricalTxType.DELEGATION) { who = translation.activitySelf; - } else if (cTx.shieldedOutputs) { - who = translation.activityShieldedAddress; + const descriptor = txSelfMap(cTx.amount, cTx.shieldAmount); + icon = descriptor.icon; + colour = descriptor.colour; + content = descriptor.content; + amountToShow = descriptor.amount; } else { - const arrAddresses = cTx.receivers + let arrAddresses = cTx.receivers .map((addr) => [wallet.isOwnAddress(addr), addr]) .filter(([isOwnAddress, _]) => { return cTx.type === HistoricalTxType.RECEIVED @@ -221,6 +233,9 @@ async function parseTXs(arrTXs) { : !isOwnAddress; }) .map(([_, addr]) => getNameOrAddress(cAccount, addr)); + if (cTx.type == HistoricalTxType.RECEIVED) { + arrAddresses = arrAddresses.concat(cTx.shieldReceivers); + } who = [ ...new Set( @@ -231,16 +246,37 @@ async function parseTXs(arrTXs) { ) ), ].join(', ') + '...'; + if ( + cTx.type == HistoricalTxType.SENT && + arrAddresses.length == 0 + ) { + // We sent a shield note to someone, but we cannot decrypt the recipient + // So show a generic "Sent to shield address" + who = translation.activityShieldedAddress; + } } content = content.replace(/{.}/, who); } + // Format the amount to reduce text size + let formattedAmt = ''; + if (amountToShow < 0.01) { + formattedAmt = beautifyNumber('0.01', '13px'); + } else if (amountToShow >= 100) { + formattedAmt = beautifyNumber( + Math.round(amountToShow).toString(), + '13px' + ); + } else { + formattedAmt = beautifyNumber(`${amountToShow.toFixed(2)}`, '13px'); + } + newTxs.push({ date: strDate, id: cTx.id, content: props.rewards ? cTx.id : content, formattedAmt, - amount: cTx.amount, + amount: amountToShow, confirmed: fConfirmed, icon, colour, diff --git a/scripts/historical_tx.js b/scripts/historical_tx.js index 90ac30c4a..83c3446f9 100644 --- a/scripts/historical_tx.js +++ b/scripts/historical_tx.js @@ -6,27 +6,34 @@ export class HistoricalTx { * @param {HistoricalTxType} type - The type of transaction. * @param {string} id - The transaction ID. * @param {Array} receivers - The list of 'output addresses'. - * @param {boolean} shieldedOutputs - If this transaction contains Shield outputs. + * @param {Array} shieldReceivers - The list of decrypted 'shield output addresses'. * @param {number} time - The block time of the transaction. * @param {number} blockHeight - The block height of the transaction. - * @param {number} amount - The amount transacted, in coins. + * @param {number} amount - The transparent amount transacted, in coins. + * @param {number} shieldAmount - The shielded amount transacted, in coins. + * @param {boolean} isToSelf - If the transaction is to self. + * @param {boolean} isConfirmed - Whether the transaction has been confirmed. */ constructor( type, id, receivers, - shieldedOutputs, + shieldReceivers, time, blockHeight, - amount + amount, + shieldAmount, + isToSelf ) { this.type = type; this.id = id; this.receivers = receivers; - this.shieldedOutputs = shieldedOutputs; + this.shieldReceivers = shieldReceivers; this.time = time; this.blockHeight = blockHeight; this.amount = amount; + this.shieldAmount = shieldAmount; + this.isToSelf = isToSelf; } } diff --git a/scripts/mempool.js b/scripts/mempool.js index f5b9f12bd..93812e118 100644 --- a/scripts/mempool.js +++ b/scripts/mempool.js @@ -117,13 +117,15 @@ export class Mempool { * @param {import('./transaction.js').Transaction} tx */ getDebit(tx) { - return tx.vin - .filter( - (input) => - this.getOutpointStatus(input.outpoint) & OutpointState.OURS - ) + const filteredVin = tx.vin.filter( + (input) => + this.getOutpointStatus(input.outpoint) & OutpointState.OURS + ); + const debit = filteredVin .map((i) => this.outpointToUTXO(i.outpoint)) .reduce((acc, u) => acc + (u?.value || 0), 0); + const ownAllVin = tx.vin.length === filteredVin.length; + return { debit, ownAllVin }; } /** @@ -133,17 +135,21 @@ export class Mempool { getCredit(tx) { const txid = tx.txid; - return tx.vout - .filter( - (_, i) => - this.getOutpointStatus( - new COutpoint({ - txid, - n: i, - }) - ) & OutpointState.OURS - ) - .reduce((acc, u) => acc + u?.value ?? 0, 0); + const filteredVout = tx.vout.filter( + (_, i) => + this.getOutpointStatus( + new COutpoint({ + txid, + n: i, + }) + ) & OutpointState.OURS + ); + const credit = filteredVout.reduce((acc, u) => acc + u?.value ?? 0, 0); + const ownAllVout = tx.vout.length === filteredVout.length; + return { + credit, + ownAllVout, + }; } /** diff --git a/scripts/reader.js b/scripts/reader.js index 0e9f74b76..7fc70595f 100644 --- a/scripts/reader.js +++ b/scripts/reader.js @@ -38,9 +38,7 @@ export class Reader { } if (done) { this.#done = true; - if (this.#awaiter) { - this.#awaiter(); - } + if (this.#awaiter) this.#awaiter(); break; } } diff --git a/scripts/utils.js b/scripts/utils.js index 2a34c1dcf..80478cfef 100644 --- a/scripts/utils.js +++ b/scripts/utils.js @@ -13,6 +13,10 @@ export function bytesToHex(bytes) { return Buffer.from(bytes).toString('hex'); } +export function reverseAndSwapEndianess(hex) { + return bytesToHex(hexToBytes(hex).reverse()); +} + /** * Double SHA256 hash a byte array * @param {Array} buff - Bytes to hash diff --git a/scripts/wallet.js b/scripts/wallet.js index 725faf1ed..e06e9d2b5 100644 --- a/scripts/wallet.js +++ b/scripts/wallet.js @@ -16,7 +16,12 @@ import { Database } from './database.js'; import { RECEIVE_TYPES } from './contacts-book.js'; import { Account } from './accounts.js'; import { fAdvancedMode } from './settings.js'; -import { bytesToHex, hexToBytes, sleep } from './utils.js'; +import { + bytesToHex, + hexToBytes, + reverseAndSwapEndianess, + sleep, +} from './utils.js'; import { strHardwareName } from './ledger.js'; import { OutpointState, Mempool } from './mempool.js'; import { getEventEmitter } from './event_bus.js'; @@ -646,20 +651,23 @@ export class Wallet { /** * Convert a list of Blockbook transactions to HistoricalTxs * @param {import('./transaction.js').Transaction[]} arrTXs - An array of the Blockbook TXs - * @returns {Array} - A new array of `HistoricalTx`-formatted transactions + * @returns {Promise>} - A new array of `HistoricalTx`-formatted transactions */ - // TODO: add shield data to txs - toHistoricalTXs(arrTXs) { + async toHistoricalTXs(arrTXs) { let histTXs = []; for (const tx of arrTXs) { + const { credit, ownAllVout } = this.#mempool.getCredit(tx); + const { debit, ownAllVin } = this.#mempool.getDebit(tx); // The total 'delta' or change in balance, from the Tx's sums - let nAmount = - (this.#mempool.getCredit(tx) - this.#mempool.getDebit(tx)) / - COIN; + let nAmount = (credit - debit) / COIN; + // Shielded data + const { shieldCredit, shieldDebit, arrShieldReceivers } = + await this.extractSaplingAmounts(tx); + const nShieldAmount = (shieldCredit - shieldDebit) / COIN; + const ownAllShield = shieldDebit - shieldCredit === tx.valueBalance; // The receiver addresses, if any let arrReceivers = this.getOutAddress(tx); - const getFilteredCredit = (filter) => { return tx.vout .filter((_, i) => { @@ -689,9 +697,9 @@ export class Wallet { return addr[0] === cChainParams.current.STAKING_PREFIX; }); nAmount = getFilteredCredit(OutpointState.P2CS) / COIN; - } else if (nAmount > 0) { + } else if (nAmount + nShieldAmount > 0) { type = HistoricalTxType.RECEIVED; - } else if (nAmount < 0) { + } else if (nAmount + nShieldAmount < 0) { type = HistoricalTxType.SENT; } @@ -700,10 +708,12 @@ export class Wallet { type, tx.txid, arrReceivers, - false, + arrShieldReceivers, tx.blockTime, tx.blockHeight, - Math.abs(nAmount) + nAmount, + nShieldAmount, + ownAllVin && ownAllVout && ownAllShield ) ); } @@ -711,10 +721,38 @@ export class Wallet { } /** + * Extract the sapling spent, received and shield addressed, regarding the wallet, from a tx + * @param {import('./transaction.js').Transaction} tx - a Transaction object + */ + async extractSaplingAmounts(tx) { + let shieldCredit = 0; + let shieldDebit = 0; + let arrShieldReceivers = []; + if (!tx.hasShieldData || !wallet.hasShield()) { + return { shieldCredit, shieldDebit, arrShieldReceivers }; + } + + for (const shieldSpend of tx.shieldSpend) { + const nullifier = reverseAndSwapEndianess(shieldSpend.nullifier); + const spentNote = this.#shield.getNoteFromNullifier(nullifier); + if (spentNote) { + shieldDebit += spentNote.value; + } + } + const myOutputNotes = await this.#shield.decryptTransactionOutputs( + tx.serialize() + ); + for (const note of myOutputNotes) { + shieldCredit += note.value; + arrShieldReceivers.push(note.recipient); + } + return { shieldCredit, shieldDebit, arrShieldReceivers }; + } + /* * @param {Transaction} tx */ - #pushToHistoricalTx(tx) { - const hTx = this.toHistoricalTXs([tx])[0]; + async #pushToHistoricalTx(tx) { + const hTx = (await this.toHistoricalTXs([tx]))[0]; this.#historicalTxs.insert(hTx); } @@ -733,8 +771,8 @@ export class Wallet { getEventEmitter().disableEvent('balance-update'); getEventEmitter().disableEvent('new-tx'); - await this.loadFromDisk(); await this.loadShieldFromDisk(); + await this.loadFromDisk(); // Let's set the last processed block 5 blocks behind the actual chain tip // This is just to be sure since blockbook (as we know) // usually does not return txs of the actual last block. @@ -819,7 +857,20 @@ export class Wallet { const start = performance.now(); // Process the current batch of blocks before starting to parse the next one if (blocksArray.length) { - await this.#shield.handleBlocks(blocksArray); + const ownTxs = await this.#shield.handleBlocks(blocksArray); + // TODO: slow! slow! slow! + if (ownTxs.length > 0) { + for (const block of blocksArray) { + for (const tx of block.txs) { + if (ownTxs.includes(tx.hex)) { + const parsed = Transaction.fromHex(tx.hex); + parsed.blockTime = block.time; + parsed.blockHeight = block.height; + await this.addTransaction(parsed); + } + } + } + } } handleBlocksTime += performance.now() - start; blocksArray = []; @@ -931,25 +982,7 @@ export class Wallet { ) { try { block = await cNet.getBlock(blockHeight); - if (block.txs) { - if ( - this.hasShield() && - blockHeight > this.#shield.getLastSyncedBlock() - ) { - await this.#shield.handleBlock(block); - } - for (const tx of block.txs) { - const parsed = Transaction.fromHex(tx.hex); - parsed.blockHeight = blockHeight; - parsed.blockTime = block.mediantime; - // Avoid wasting memory on txs that do not regard our wallet - if (this.ownTransaction(parsed)) { - await this.addTransaction(parsed); - } - } - } else { - break; - } + await this.#handleBlock(block, blockHeight); this.#lastProcessedBlock = blockHeight; } catch (e) { debugError(DebugTopics.WALLET, e); @@ -971,15 +1004,15 @@ export class Wallet { ); async #checkShieldSaplingRoot(networkSaplingRoot) { - const saplingRoot = bytesToHex( - hexToBytes(await this.#shield.getSaplingRoot()).reverse() + const saplingRoot = reverseAndSwapEndianess( + await this.#shield.getSaplingRoot() ); // If explorer sapling root is different from ours, there must be a sync error if (saplingRoot !== networkSaplingRoot) { createAlert('warning', translation.badSaplingRoot, 5000); this.#mempool = new Mempool(); - this.#isSynced = false; await this.#resetShield(); + this.#isSynced = false; await this.#transparentSync(); await this.#syncShield(); return false; @@ -1026,6 +1059,7 @@ export class Wallet { ); await this.#resetShield(); } + return; } async #resetShield() { @@ -1268,14 +1302,39 @@ export class Wallet { const db = await Database.getInstance(); await db.storeTx(transaction); } - if (tx) { this.#historicalTxs.remove((hTx) => hTx.id === tx.txid); } - this.#pushToHistoricalTx(transaction); + await this.#pushToHistoricalTx(transaction); getEventEmitter().emit('new-tx'); } + /** + * Handle the various transactions of a block + * @param block - block outputted from any PIVX node + * @param {number} blockHeight - the height of the block in the chain + * @param {boolean} allowOwn - whether to add transaction that satisfy ownTransaction() + */ + async #handleBlock(block, blockHeight, allowOwn = true) { + let shieldTxs = []; + if ( + this.hasShield() && + blockHeight > this.#shield.getLastSyncedBlock() + ) { + shieldTxs = await this.#shield.handleBlock(block); + } + for (const tx of block.txs) { + const parsed = Transaction.fromHex(tx.hex); + parsed.blockHeight = blockHeight; + parsed.blockTime = block.time; + // Avoid wasting memory on txs that do not regard our wallet + const isOwned = allowOwn ? this.ownTransaction(parsed) : false; + if (isOwned || shieldTxs.includes(tx.hex)) { + await this.addTransaction(parsed); + } + } + } + /** * Check if any vin or vout of the transaction belong to the wallet * @param {import('./transaction.js').Transaction} transaction diff --git a/tests/unit/mempool.spec.js b/tests/unit/mempool.spec.js index 2294f7883..f41370cca 100644 --- a/tests/unit/mempool.spec.js +++ b/tests/unit/mempool.spec.js @@ -121,20 +121,20 @@ describe('mempool tests', () => { vout: [], }); mempool.addTransaction(spendTx); - expect(mempool.getDebit(spendTx)).toBe(5000000 + 4992400); + expect(mempool.getDebit(spendTx).debit).toBe(5000000 + 4992400); - expect(mempool.getDebit(new Transaction())).toBe(0); + expect(mempool.getDebit(new Transaction()).debit).toBe(0); }); it('gives correct credit', () => { - expect(mempool.getCredit(tx)).toBe(5000000 + 4992400); + expect(mempool.getCredit(tx).credit).toBe(5000000 + 4992400); // Result should stay the same even if the UTXOs are spent mempool.setSpent(new COutpoint({ txid: tx.txid, n: 1 })); - expect(mempool.getCredit(tx)).toBe(5000000 + 4992400); + expect(mempool.getCredit(tx).credit).toBe(5000000 + 4992400); mempool.setSpent(new COutpoint({ txid: tx.txid, n: 0 })); - expect(mempool.getCredit(tx)).toBe(5000000 + 4992400); - expect(mempool.getCredit(new Transaction())).toBe(0); + expect(mempool.getCredit(tx).credit).toBe(5000000 + 4992400); + expect(mempool.getCredit(new Transaction()).credit).toBe(0); }); it('marks outpoint as spent correctly', () => {