From aa88edbe0c3c1e6f6f63e1205d9bbd5ca33f1e1c Mon Sep 17 00:00:00 2001 From: junderw Date: Tue, 7 May 2024 22:19:02 +0900 Subject: [PATCH 1/4] Fix Testnet4 addition --- src/chain.rs | 15 ++++++++++++--- src/daemon.rs | 1 + src/elements/peg.rs | 8 +++++++- src/rest.rs | 5 ++++- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/chain.rs b/src/chain.rs index 8abf9a4a..ccb2b353 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -128,7 +128,7 @@ impl Network { pub fn genesis_hash(network: Network) -> BlockHash { #[cfg(not(feature = "liquid"))] - return bitcoin_genesis_hash(network.into()); + return bitcoin_genesis_hash(network); #[cfg(feature = "liquid")] return liquid_genesis_hash(network); } @@ -139,13 +139,16 @@ pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash { genesis_block(BNetwork::Bitcoin).block_hash(); static ref TESTNET_GENESIS: bitcoin::BlockHash = genesis_block(BNetwork::Testnet).block_hash(); - static ref TESTNET4_GENESIS: bitcoin::BlockHash = - BlockHash::from_str("00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043").unwrap(); + static ref TESTNET4_GENESIS: bitcoin::BlockHash = bitcoin::BlockHash::from_str( + "00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043" + ) + .unwrap(); static ref REGTEST_GENESIS: bitcoin::BlockHash = genesis_block(BNetwork::Regtest).block_hash(); static ref SIGNET_GENESIS: bitcoin::BlockHash = genesis_block(BNetwork::Signet).block_hash(); } + #[cfg(not(feature = "liquid"))] match network { Network::Bitcoin => *BITCOIN_GENESIS, Network::Testnet => *TESTNET_GENESIS, @@ -153,6 +156,12 @@ pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash { Network::Regtest => *REGTEST_GENESIS, Network::Signet => *SIGNET_GENESIS, } + #[cfg(feature = "liquid")] + match network { + Network::Liquid => *BITCOIN_GENESIS, + Network::LiquidTestnet => *TESTNET_GENESIS, + Network::LiquidRegtest => *REGTEST_GENESIS, + } } #[cfg(feature = "liquid")] diff --git a/src/daemon.rs b/src/daemon.rs index b8bde690..f04045d0 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -295,6 +295,7 @@ pub struct Daemon { } impl Daemon { + #[allow(clippy::too_many_arguments)] pub fn new( daemon_dir: PathBuf, blocks_dir: PathBuf, diff --git a/src/elements/peg.rs b/src/elements/peg.rs index cd339e60..7956973f 100644 --- a/src/elements/peg.rs +++ b/src/elements/peg.rs @@ -19,7 +19,13 @@ pub fn get_pegout_data( let pegged_asset_id = network.pegged_asset()?; txout.pegout_data().filter(|pegout| { pegout.asset == Asset::Explicit(*pegged_asset_id) - && pegout.genesis_hash == bitcoin_genesis_hash(parent_network) + && pegout.genesis_hash + == bitcoin_genesis_hash(match parent_network { + BNetwork::Bitcoin => Network::Liquid, + BNetwork::Testnet => Network::LiquidTestnet, + BNetwork::Signet => return false, + BNetwork::Regtest => Network::LiquidRegtest, + }) }) } diff --git a/src/rest.rs b/src/rest.rs index 2e061b11..ce8bef96 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -1668,7 +1668,10 @@ fn address_to_scripthash(addr: &str, network: Network) -> Result Date: Tue, 7 May 2024 14:48:01 +0900 Subject: [PATCH 2/4] Feat: Order history entries in the order they appear in block. --- src/elements/asset.rs | 11 +++++----- src/new_index/db.rs | 6 +++++- src/new_index/schema.rs | 46 ++++++++++++++++++++++++++++++++++------- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/elements/asset.rs b/src/elements/asset.rs index b6cd704f..6eeed728 100644 --- a/src/elements/asset.rs +++ b/src/elements/asset.rs @@ -174,17 +174,16 @@ pub struct BurningInfo { pub fn index_confirmed_tx_assets( tx: &Transaction, confirmed_height: u32, + tx_position: u32, network: Network, parent_network: BNetwork, rows: &mut Vec, ) { let (history, issuances) = index_tx_assets(tx, network, parent_network); - rows.extend( - history.into_iter().map(|(asset_id, info)| { - asset_history_row(&asset_id, confirmed_height, info).into_row() - }), - ); + rows.extend(history.into_iter().map(|(asset_id, info)| { + asset_history_row(&asset_id, confirmed_height, tx_position, info).into_row() + })); // the initial issuance is kept twice: once in the history index under I, // and once separately under i for asset lookup with some more associated metadata. @@ -336,12 +335,14 @@ fn index_tx_assets( fn asset_history_row( asset_id: &AssetId, confirmed_height: u32, + tx_position: u32, txinfo: TxHistoryInfo, ) -> TxHistoryRow { let key = TxHistoryKey { code: b'I', hash: full_hash(&asset_id.into_inner()[..]), confirmed_height, + tx_position, txinfo, }; TxHistoryRow { key } diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 5e4b37a1..4fe9020c 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -5,7 +5,11 @@ use std::path::Path; use crate::config::Config; use crate::util::{bincode_util, Bytes}; -static DB_VERSION: u32 = 1; +/// Each version will break any running instance with a DB that has a differing version. +/// It will also break if light mode is enabled or disabled. +// 1 = Original DB (since fork from Blockstream) +// 2 = Add tx position to TxHistory rows and place Spending before Funding +static DB_VERSION: u32 = 2; #[derive(Debug, Eq, PartialEq)] pub struct DBRow { diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 00ee3e89..bcee4793 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -1246,9 +1246,16 @@ fn index_blocks( .par_iter() // serialization is CPU-intensive .map(|b| { let mut rows = vec![]; - for tx in &b.block.txdata { + for (idx, tx) in b.block.txdata.iter().enumerate() { let height = b.entry.height() as u32; - index_transaction(tx, height, previous_txos_map, &mut rows, iconfig); + index_transaction( + tx, + height, + idx as u32, + previous_txos_map, + &mut rows, + iconfig, + ); } rows.push(BlockRow::new_done(full_hash(&b.entry.hash()[..])).into_row()); // mark block as "indexed" rows @@ -1261,6 +1268,7 @@ fn index_blocks( fn index_transaction( tx: &Transaction, confirmed_height: u32, + tx_position: u32, previous_txos_map: &HashMap, rows: &mut Vec, iconfig: &IndexerConfig, @@ -1276,6 +1284,7 @@ fn index_transaction( let history = TxHistoryRow::new( &txo.script_pubkey, confirmed_height, + tx_position, TxHistoryInfo::Funding(FundingInfo { txid, vout: txo_index as u16, @@ -1302,6 +1311,7 @@ fn index_transaction( let history = TxHistoryRow::new( &prev_txo.script_pubkey, confirmed_height, + tx_position, TxHistoryInfo::Spending(SpendingInfo { txid, vin: txi_index as u16, @@ -1326,6 +1336,7 @@ fn index_transaction( asset::index_confirmed_tx_assets( tx, confirmed_height, + tx_position, iconfig.network, iconfig.parent_network, rows, @@ -1567,8 +1578,11 @@ pub struct SpendingInfo { #[derive(Serialize, Deserialize, Debug)] #[cfg_attr(test, derive(PartialEq, Eq))] pub enum TxHistoryInfo { - Funding(FundingInfo), + // If a spend and a fund for the same scripthash + // occur in the same tx, spends should come first. + // This ordering comes from the enum order. Spending(SpendingInfo), + Funding(FundingInfo), #[cfg(feature = "liquid")] Issuing(asset::IssuingInfo), @@ -1602,6 +1616,7 @@ pub struct TxHistoryKey { pub code: u8, // H for script history or I for asset history (elements only) pub hash: FullHash, // either a scripthash (always on bitcoin) or an asset id (elements only) pub confirmed_height: u32, // MUST be serialized as big-endian (for correct scans). + pub tx_position: u32, // MUST be serialized as big-endian (for correct scans). Position in block. pub txinfo: TxHistoryInfo, } @@ -1610,11 +1625,17 @@ pub struct TxHistoryRow { } impl TxHistoryRow { - fn new(script: &Script, confirmed_height: u32, txinfo: TxHistoryInfo) -> Self { + fn new( + script: &Script, + confirmed_height: u32, + tx_position: u32, + txinfo: TxHistoryInfo, + ) -> Self { let key = TxHistoryKey { code: b'H', hash: compute_script_hash(script), confirmed_height, + tx_position, txinfo, }; TxHistoryRow { key } @@ -1845,8 +1866,10 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // confirmed_height 0, 0, 0, 2, + // tx_position + 0, 0, 0, 3, // TxHistoryInfo variant (Funding) - 0, 0, 0, 0, + 0, 0, 0, 1, // FundingInfo // txid 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -1865,7 +1888,8 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 2, - 0, 0, 0, 0, + 0, 0, 0, 3, + 0, 0, 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 3, @@ -1879,7 +1903,8 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 2, - 0, 0, 0, 1, + 0, 0, 0, 3, + 0, 0, 0, 0, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 0, 12, @@ -1895,7 +1920,8 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 2, - 0, 0, 0, 1, + 0, 0, 0, 3, + 0, 0, 0, 0, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 0, 12, @@ -1912,6 +1938,7 @@ mod tests { code: b'H', hash: [1; 32], confirmed_height: 2, + tx_position: 3, txinfo: super::TxHistoryInfo::Funding(super::FundingInfo { txid: [2; 32], vout: 3, @@ -1924,6 +1951,7 @@ mod tests { code: b'H', hash: [1; 32], confirmed_height: 2, + tx_position: 3, txinfo: super::TxHistoryInfo::Funding(super::FundingInfo { txid: [2; 32], vout: 3, @@ -1936,6 +1964,7 @@ mod tests { code: b'H', hash: [1; 32], confirmed_height: 2, + tx_position: 3, txinfo: super::TxHistoryInfo::Spending(super::SpendingInfo { txid: [18; 32], vin: 12, @@ -1950,6 +1979,7 @@ mod tests { code: b'H', hash: [1; 32], confirmed_height: 2, + tx_position: 3, txinfo: super::TxHistoryInfo::Spending(super::SpendingInfo { txid: [18; 32], vin: 12, From 3dab4c7eb96a3e831aba0a2f0198adb59a6bfc80 Mon Sep 17 00:00:00 2001 From: junderw Date: Fri, 10 May 2024 06:17:12 +0900 Subject: [PATCH 3/4] Use u16 for tx position --- src/elements/asset.rs | 4 ++-- src/new_index/schema.rs | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/elements/asset.rs b/src/elements/asset.rs index 6eeed728..01873a30 100644 --- a/src/elements/asset.rs +++ b/src/elements/asset.rs @@ -174,7 +174,7 @@ pub struct BurningInfo { pub fn index_confirmed_tx_assets( tx: &Transaction, confirmed_height: u32, - tx_position: u32, + tx_position: u16, network: Network, parent_network: BNetwork, rows: &mut Vec, @@ -335,7 +335,7 @@ fn index_tx_assets( fn asset_history_row( asset_id: &AssetId, confirmed_height: u32, - tx_position: u32, + tx_position: u16, txinfo: TxHistoryInfo, ) -> TxHistoryRow { let key = TxHistoryKey { diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index bcee4793..b0a7998e 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -1251,7 +1251,7 @@ fn index_blocks( index_transaction( tx, height, - idx as u32, + idx as u16, previous_txos_map, &mut rows, iconfig, @@ -1268,14 +1268,14 @@ fn index_blocks( fn index_transaction( tx: &Transaction, confirmed_height: u32, - tx_position: u32, + tx_position: u16, previous_txos_map: &HashMap, rows: &mut Vec, iconfig: &IndexerConfig, ) { // persist history index: - // H{funding-scripthash}{funding-height}F{funding-txid:vout} → "" - // H{funding-scripthash}{spending-height}S{spending-txid:vin}{funding-txid:vout} → "" + // H{funding-scripthash}{spending-height}{spending-block-pos}S{spending-txid:vin}{funding-txid:vout} → "" + // H{funding-scripthash}{funding-height}{funding-block-pos}F{funding-txid:vout} → "" // persist "edges" for fast is-this-TXO-spent check // S{funding-txid:vout}{spending-txid:vin} → "" let txid = full_hash(&tx.txid()[..]); @@ -1616,7 +1616,7 @@ pub struct TxHistoryKey { pub code: u8, // H for script history or I for asset history (elements only) pub hash: FullHash, // either a scripthash (always on bitcoin) or an asset id (elements only) pub confirmed_height: u32, // MUST be serialized as big-endian (for correct scans). - pub tx_position: u32, // MUST be serialized as big-endian (for correct scans). Position in block. + pub tx_position: u16, // MUST be serialized as big-endian (for correct scans). Position in block. pub txinfo: TxHistoryInfo, } @@ -1628,7 +1628,7 @@ impl TxHistoryRow { fn new( script: &Script, confirmed_height: u32, - tx_position: u32, + tx_position: u16, txinfo: TxHistoryInfo, ) -> Self { let key = TxHistoryKey { @@ -1867,7 +1867,7 @@ mod tests { // confirmed_height 0, 0, 0, 2, // tx_position - 0, 0, 0, 3, + 0, 3, // TxHistoryInfo variant (Funding) 0, 0, 0, 1, // FundingInfo @@ -1888,7 +1888,7 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 2, - 0, 0, 0, 3, + 0, 3, 0, 0, 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -1903,7 +1903,7 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 2, - 0, 0, 0, 3, + 0, 3, 0, 0, 0, 0, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, @@ -1920,7 +1920,7 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 2, - 0, 0, 0, 3, + 0, 3, 0, 0, 0, 0, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, From f7b77a3db0710ac97fd64916755c35c84de3f42e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 13 May 2024 18:42:08 +0000 Subject: [PATCH 4/4] Fix tx order in address summaries --- src/new_index/schema.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index b0a7998e..eba60588 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -533,23 +533,23 @@ impl ChainQuery { let rows = self .history_iter_scan_reverse(code, hash) .map(TxHistoryRow::from_row) - .map(|row| (row.get_txid(), row.key.txinfo)) - .skip_while(|(txid, _)| { + .map(|row| (row.get_txid(), row.key.txinfo, row.key.tx_position)) + .skip_while(|(txid, _, _)| { // skip until we reach the last_seen_txid last_seen_txid.map_or(false, |last_seen_txid| last_seen_txid != txid) }) - .skip_while(|(txid, _)| { + .skip_while(|(txid, _, _)| { // skip the last_seen_txid itself last_seen_txid.map_or(false, |last_seen_txid| last_seen_txid == txid) }) - .filter_map(|(txid, info)| { + .filter_map(|(txid, info, tx_position)| { self.tx_confirming_block(&txid) - .map(|b| (txid, info, b.height, b.time)) + .map(|b| (txid, info, b.height, b.time, tx_position)) }); // collate utxo funding/spending events by transaction let mut map: HashMap = HashMap::new(); - for (txid, info, height, time) in rows { + for (txid, info, height, time, tx_position) in rows { if !map.contains_key(&txid) && map.len() == limit { break; } @@ -565,6 +565,7 @@ impl ChainQuery { value: info.value.try_into().unwrap_or(0), height, time, + tx_position, }); } #[cfg(not(feature = "liquid"))] @@ -578,6 +579,7 @@ impl ChainQuery { value: 0_i64.saturating_sub(info.value.try_into().unwrap_or(0)), height, time, + tx_position, }); } #[cfg(feature = "liquid")] @@ -587,6 +589,7 @@ impl ChainQuery { value: 0, height, time, + tx_position, }); } #[cfg(feature = "liquid")] @@ -596,6 +599,7 @@ impl ChainQuery { value: 0, height, time, + tx_position, }); } #[cfg(feature = "liquid")] @@ -606,7 +610,11 @@ impl ChainQuery { let mut tx_summaries = map.into_values().collect::>(); tx_summaries.sort_by(|a, b| { if a.height == b.height { - a.value.cmp(&b.value) + if a.tx_position == b.tx_position { + a.value.cmp(&b.value) + } else { + b.tx_position.cmp(&a.tx_position) + } } else { b.height.cmp(&a.height) } @@ -1702,6 +1710,7 @@ pub struct TxHistorySummary { height: usize, value: i64, time: u32, + tx_position: u16, } #[derive(Serialize, Deserialize)]