From 0a2cff40503770308f8a36a158ceb7651c144738 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 24 Jul 2023 17:06:30 +0900 Subject: [PATCH 1/8] Add bulk block transactions endpoint --- src/new_index/schema.rs | 28 ++++++++++++++++++++++++++++ src/rest.rs | 12 ++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index a0f9ca812..c08370541 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -371,6 +371,34 @@ impl ChainQuery { } } + pub fn get_block_txs(&self, hash: &BlockHash) -> Option> { + let _timer = self.start_timer("get_block_txs"); + + let txids: Option> = if self.light_mode { + // TODO fetch block as binary from REST API instead of as hex + let mut blockinfo = self.daemon.getblock_raw(hash, 1).ok()?; + Some(serde_json::from_value(blockinfo["tx"].take()).unwrap()) + } else { + self.store + .txstore_db + .get(&BlockRow::txids_key(full_hash(&hash[..]))) + .map(|val| bincode::deserialize(&val).expect("failed to parse block txids")) + }; + + txids.and_then(|txid_vec| { + let mut transactions = Vec::new(); + + for txid in txid_vec { + match self.lookup_txn(&txid, Some(hash)) { + Some(transaction) => transactions.push(transaction), + None => return None, + } + } + + Some(transactions) + }) + } + pub fn get_block_meta(&self, hash: &BlockHash) -> Option { let _timer = self.start_timer("get_block_meta"); diff --git a/src/rest.rs b/src/rest.rs index 19b8dd6f4..ceff090e8 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -699,6 +699,18 @@ fn handle_request( .ok_or_else(|| HttpError::not_found("Block not found".to_string()))?; json_response(txids, TTL_LONG) } + (&Method::GET, Some(&"block"), Some(hash), Some(&"txs"), None, None) => { + let hash = BlockHash::from_hex(hash)?; + let txs = query + .chain() + .get_block_txs(&hash) + .ok_or_else(|| HttpError::not_found("Block not found".to_string()))? + .into_iter() + .map(|tx| (tx, None)) + .collect(); + + json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + } (&Method::GET, Some(&"block"), Some(hash), Some(&"header"), None, None) => { let hash = BlockHash::from_hex(hash)?; let header = query From d688a8b08cb8ece0f7ab4f9f33b5c4d972c5b24d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 25 Jul 2023 13:28:24 +0900 Subject: [PATCH 2/8] pre-allocate block txs vec --- src/new_index/schema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index c08370541..fb39b9317 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -386,7 +386,7 @@ impl ChainQuery { }; txids.and_then(|txid_vec| { - let mut transactions = Vec::new(); + let mut transactions = Vec::with_capacity(txid_vec.len()); for txid in txid_vec { match self.lookup_txn(&txid, Some(hash)) { From 96e4324ada1fab2905df3b8cd40fcf725aa909af Mon Sep 17 00:00:00 2001 From: junderw Date: Tue, 25 Jul 2023 19:25:51 -0700 Subject: [PATCH 3/8] Fix: mempool.update() frequent crashes in the main loop --- src/new_index/mempool.rs | 96 ++++++++++++++++++++++++++-------------- src/new_index/query.rs | 15 +++++-- src/new_index/schema.rs | 2 + src/rest.rs | 74 +++++++++++++++---------------- 4 files changed, 113 insertions(+), 74 deletions(-) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 863d8eece..bdaaaa494 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -206,9 +206,7 @@ impl Mempool { TxHistoryInfo::Funding(info) => { // Liquid requires some additional information from the txo that's not available in the TxHistoryInfo index. #[cfg(feature = "liquid")] - let txo = self - .lookup_txo(&entry.get_funded_outpoint()) - .expect("missing txo"); + let txo = self.lookup_txo(&entry.get_funded_outpoint()); Some(Utxo { txid: deserialize(&info.txid).expect("invalid txid"), @@ -345,7 +343,7 @@ impl Mempool { } }; // Add new transactions - self.add(to_add)?; + self.add(to_add); // Remove missing transactions self.remove(to_remove); @@ -370,39 +368,64 @@ impl Mempool { pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Txid) -> Result<()> { if self.txstore.get(txid).is_none() { if let Ok(tx) = daemon.getmempooltx(txid) { - self.add(vec![tx])?; + if self.add(vec![tx]) == 0 { + return Err(format!( + "Unable to add {txid} to mempool likely due to missing parents." + ) + .into()); + } } } - Ok(()) } - fn add(&mut self, txs: Vec) -> Result<()> { + /// Add transactions to the mempool. + /// + /// The return value is the number of transactions processed. + fn add(&mut self, txs: Vec) -> usize { self.delta .with_label_values(&["add"]) .observe(txs.len() as f64); let _timer = self.latency.with_label_values(&["add"]).start_timer(); - - let mut txids = vec![]; - // Phase 1: add to txstore - for tx in txs { - let txid = tx.txid(); - txids.push(txid); - self.txstore.insert(txid, tx); + let txlen = txs.len(); + if txlen == 0 { + return 0; } - // Phase 2: index history and spend edges (can fail if some txos cannot be found) - let txos = match self.lookup_txos(&self.get_prevouts(&txids)) { - Ok(txos) => txos, - Err(err) => { - warn!("lookup txouts failed: {}", err); - // TODO: should we remove txids from txstore? - return Ok(()); - } - }; - for txid in txids { - let tx = self.txstore.get(&txid).expect("missing mempool tx"); + debug!("Adding {} transactions to Mempool", txlen); + + // Phase 1: index history and spend edges (some txos can be missing) + let txids: Vec<_> = txs.iter().map(Transaction::txid).collect(); + let txos = self.lookup_txos(&self.get_prevouts(&txids)); + + // Count how many transactions were actually processed. + let mut processed_count = 0; + + // Phase 2: Iterate over the transactions and do the following: + // 1. Find all of the TxOuts of each input parent using `txos` + // 2. If any parent wasn't found, skip parsing this transaction + // 3. Insert TxFeeInfo into info. + // 4. Push TxOverview into recent tx queue. + // 5. Create the Spend and Fund TxHistory structs for inputs + outputs + // 6. Insert all TxHistory into history. + // 7. Insert the tx edges into edges (HashMap of (Outpoint, (Txid, vin))) + // 8. (Liquid only) Parse assets of tx. + for owned_tx in txs { + let txid = owned_tx.txid(); + + let entry = self.txstore.entry(txid); + // Note: This fn doesn't overwrite existing transactions, + // But that's ok, we didn't insert unrelated txes + // into any given txid anyways, so this will always insert. + let tx = &*entry.or_insert_with(|| owned_tx); + + let prevouts = match extract_tx_prevouts(tx, &txos, false) { + Ok(v) => v, + Err(e) => { + warn!("Skipping tx {txid} missing parent error: {e}"); + continue; + } + }; let txid_bytes = full_hash(&txid[..]); - let prevouts = extract_tx_prevouts(tx, &txos, false)?; // Get feeinfo for caching and recent tx overview let feeinfo = TxFeeInfo::new(tx, &prevouts, self.config.network_type); @@ -472,18 +495,20 @@ impl Mempool { &mut self.asset_history, &mut self.asset_issuance, ); + + processed_count += 1; } - Ok(()) + processed_count } - pub fn lookup_txo(&self, outpoint: &OutPoint) -> Result { + pub fn lookup_txo(&self, outpoint: &OutPoint) -> TxOut { let mut outpoints = BTreeSet::new(); outpoints.insert(*outpoint); - Ok(self.lookup_txos(&outpoints)?.remove(outpoint).unwrap()) + self.lookup_txos(&outpoints).remove(outpoint).unwrap() } - pub fn lookup_txos(&self, outpoints: &BTreeSet) -> Result> { + pub fn lookup_txos(&self, outpoints: &BTreeSet) -> HashMap { let _timer = self .latency .with_label_values(&["lookup_txos"]) @@ -494,18 +519,21 @@ impl Mempool { let mempool_txos = outpoints .iter() .filter(|outpoint| !confirmed_txos.contains_key(outpoint)) - .map(|outpoint| { + .flat_map(|outpoint| { self.txstore .get(&outpoint.txid) .and_then(|tx| tx.output.get(outpoint.vout as usize).cloned()) .map(|txout| (*outpoint, txout)) - .chain_err(|| format!("missing outpoint {:?}", outpoint)) + .or_else(|| { + warn!("missing outpoint {:?}", outpoint); + None + }) }) - .collect::>>()?; + .collect::>(); let mut txos = confirmed_txos; txos.extend(mempool_txos); - Ok(txos) + txos } fn get_prevouts(&self, txids: &[Txid]) -> BTreeSet { diff --git a/src/new_index/query.rs b/src/new_index/query.rs index e95e49cac..604dc61f1 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -71,10 +71,19 @@ impl Query { pub fn broadcast_raw(&self, txhex: &str) -> Result { let txid = self.daemon.broadcast_raw(txhex)?; - self.mempool + // The important part is whether we succeeded in broadcasting. + // Ignore errors in adding to the cache and show an internal warning. + if let Err(e) = self + .mempool .write() .unwrap() - .add_by_txid(&self.daemon, &txid)?; + .add_by_txid(&self.daemon, &txid) + { + warn!( + "broadcast_raw of {txid} succeeded to broadcast \ + but failed to add to mempool-electrs Mempool cache: {e}" + ); + } Ok(txid) } @@ -118,7 +127,7 @@ impl Query { .or_else(|| self.mempool().lookup_raw_txn(txid)) } - pub fn lookup_txos(&self, outpoints: &BTreeSet) -> Result> { + pub fn lookup_txos(&self, outpoints: &BTreeSet) -> HashMap { // the mempool lookup_txos() internally looks up confirmed txos as well self.mempool().lookup_txos(outpoints) } diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index fb39b9317..97d257378 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -292,6 +292,7 @@ impl Indexer { } fn add(&self, blocks: &[BlockEntry]) { + debug!("Adding {} blocks to Indexer", blocks.len()); // TODO: skip orphaned blocks? let rows = { let _timer = self.start_timer("add_process"); @@ -310,6 +311,7 @@ impl Indexer { } fn index(&self, blocks: &[BlockEntry]) { + debug!("Indexing {} blocks with Indexer", blocks.len()); let previous_txos_map = { let _timer = self.start_timer("index_lookup"); lookup_txos(&self.store.txstore_db, &get_previous_txos(blocks), false) diff --git a/src/rest.rs b/src/rest.rs index ceff090e8..bbc14ae68 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -502,7 +502,7 @@ fn prepare_txs( txs: Vec<(Transaction, Option)>, query: &Query, config: &Config, -) -> Result, errors::Error> { +) -> Vec { let outpoints = txs .iter() .flat_map(|(tx, _)| { @@ -513,12 +513,11 @@ fn prepare_txs( }) .collect(); - let prevouts = query.lookup_txos(&outpoints)?; + let prevouts = query.lookup_txos(&outpoints); - Ok(txs - .into_iter() + txs.into_iter() .map(|(tx, blockid)| TransactionValue::new(tx, blockid, &prevouts, config)) - .collect()) + .collect() } #[tokio::main] @@ -709,7 +708,7 @@ fn handle_request( .map(|tx| (tx, None)) .collect(); - json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(prepare_txs(txs, query, config), TTL_SHORT) } (&Method::GET, Some(&"block"), Some(hash), Some(&"header"), None, None) => { let hash = BlockHash::from_hex(hash)?; @@ -786,7 +785,7 @@ fn handle_request( // XXX orphraned blocks alway get TTL_SHORT let ttl = ttl_by_depth(confirmed_blockid.map(|b| b.height), query); - json_maybe_error_response(prepare_txs(txs, query, config), ttl) + json_response(prepare_txs(txs, query, config), ttl) } (&Method::GET, Some(script_type @ &"address"), Some(script_str), None, None, None) | (&Method::GET, Some(script_type @ &"scripthash"), Some(script_str), None, None, None) => { @@ -874,7 +873,7 @@ fn handle_request( ); } - json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(prepare_txs(txs, query, config), TTL_SHORT) } ( @@ -907,7 +906,7 @@ fn handle_request( .map(|(tx, blockid)| (tx, Some(blockid))) .collect(); - json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(prepare_txs(txs, query, config), TTL_SHORT) } ( &Method::GET, @@ -938,7 +937,7 @@ fn handle_request( .map(|tx| (tx, None)) .collect(); - json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(prepare_txs(txs, query, config), TTL_SHORT) } ( @@ -981,9 +980,10 @@ fn handle_request( let blockid = query.chain().tx_confirming_block(&hash); let ttl = ttl_by_depth(blockid.as_ref().map(|b| b.height), query); - let tx = prepare_txs(vec![(tx, blockid)], query, config).map(|mut v| v.remove(0)); + let mut tx = prepare_txs(vec![(tx, blockid)], query, config); + tx.remove(0); - json_maybe_error_response(tx, ttl) + json_response(tx, ttl) } (&Method::GET, Some(&"tx"), Some(hash), Some(out_type @ &"hex"), None, None) | (&Method::GET, Some(&"tx"), Some(hash), Some(out_type @ &"raw"), None, None) => { @@ -1106,7 +1106,7 @@ fn handle_request( .map(|tx| (tx, None)) .collect(); - json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(prepare_txs(txs, query, config), TTL_SHORT) } (&Method::GET, Some(&"mempool"), Some(&"txs"), last_seen_txid, None, None) => { let last_seen_txid = last_seen_txid.and_then(|txid| Txid::from_hex(txid).ok()); @@ -1117,7 +1117,7 @@ fn handle_request( .map(|tx| (tx, None)) .collect(); - json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(prepare_txs(txs, query, config), TTL_SHORT) } (&Method::GET, Some(&"mempool"), Some(&"recent"), None, None, None) => { let mempool = query.mempool(); @@ -1188,7 +1188,7 @@ fn handle_request( .map(|(tx, blockid)| (tx, Some(blockid))), ); - json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(prepare_txs(txs, query, config), TTL_SHORT) } #[cfg(feature = "liquid")] @@ -1214,7 +1214,7 @@ fn handle_request( .map(|(tx, blockid)| (tx, Some(blockid))) .collect(); - json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(prepare_txs(txs, query, config), TTL_SHORT) } #[cfg(feature = "liquid")] @@ -1228,7 +1228,7 @@ fn handle_request( .map(|tx| (tx, None)) .collect(); - json_maybe_error_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(prepare_txs(txs, query, config), TTL_SHORT) } #[cfg(feature = "liquid")] @@ -1281,26 +1281,26 @@ fn json_response(value: T, ttl: u32) -> Result, Htt .unwrap()) } -fn json_maybe_error_response( - value: Result, - ttl: u32, -) -> Result, HttpError> { - let response = Response::builder() - .header("Content-Type", "application/json") - .header("Cache-Control", format!("public, max-age={:}", ttl)) - .header("X-Powered-By", &**VERSION_STRING); - Ok(match value { - Ok(v) => response - .body(Body::from(serde_json::to_string(&v)?)) - .expect("Valid http response"), - Err(e) => response - .status(500) - .body(Body::from(serde_json::to_string( - &json!({ "error": e.to_string() }), - )?)) - .expect("Valid http response"), - }) -} +// fn json_maybe_error_response( +// value: Result, +// ttl: u32, +// ) -> Result, HttpError> { +// let response = Response::builder() +// .header("Content-Type", "application/json") +// .header("Cache-Control", format!("public, max-age={:}", ttl)) +// .header("X-Powered-By", &**VERSION_STRING); +// Ok(match value { +// Ok(v) => response +// .body(Body::from(serde_json::to_string(&v)?)) +// .expect("Valid http response"), +// Err(e) => response +// .status(500) +// .body(Body::from(serde_json::to_string( +// &json!({ "error": e.to_string() }), +// )?)) +// .expect("Valid http response"), +// }) +// } fn blocks( query: &Query, From 1b4f8466e93b2e959773625a81d6b032321f7e1c Mon Sep 17 00:00:00 2001 From: junderw Date: Tue, 25 Jul 2023 20:47:31 -0700 Subject: [PATCH 4/8] Fix unwrap that can now possibly be None --- src/new_index/mempool.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index bdaaaa494..f4b540a3f 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -206,7 +206,7 @@ impl Mempool { TxHistoryInfo::Funding(info) => { // Liquid requires some additional information from the txo that's not available in the TxHistoryInfo index. #[cfg(feature = "liquid")] - let txo = self.lookup_txo(&entry.get_funded_outpoint()); + let txo = self.lookup_txo(&entry.get_funded_outpoint())?; Some(Utxo { txid: deserialize(&info.txid).expect("invalid txid"), @@ -502,12 +502,18 @@ impl Mempool { processed_count } - pub fn lookup_txo(&self, outpoint: &OutPoint) -> TxOut { + /// Returns None if the lookup fails (mempool transaction RBF-ed etc.) + pub fn lookup_txo(&self, outpoint: &OutPoint) -> Option { let mut outpoints = BTreeSet::new(); outpoints.insert(*outpoint); - self.lookup_txos(&outpoints).remove(outpoint).unwrap() + // This can possibly be None now + self.lookup_txos(&outpoints).remove(outpoint) } + /// For a given set of OutPoints, return a HashMap + /// + /// Not all OutPoints from mempool transactions are guaranteed to be there. + /// Ensure you deal with the None case in your logic. pub fn lookup_txos(&self, outpoints: &BTreeSet) -> HashMap { let _timer = self .latency From 875057e129ee5f890b9dcacc8369aadf0e31c88f Mon Sep 17 00:00:00 2001 From: junderw Date: Tue, 25 Jul 2023 20:53:55 -0700 Subject: [PATCH 5/8] Add must_use --- src/new_index/mempool.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index f4b540a3f..b896cb1ba 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -343,7 +343,9 @@ impl Mempool { } }; // Add new transactions - self.add(to_add); + if to_add.len() > self.add(to_add) { + debug!("Mempool update added less transactions than expected"); + } // Remove missing transactions self.remove(to_remove); @@ -382,6 +384,7 @@ impl Mempool { /// Add transactions to the mempool. /// /// The return value is the number of transactions processed. + #[must_use = "Must deal with [[input vec's length]] > [[result]]."] fn add(&mut self, txs: Vec) -> usize { self.delta .with_label_values(&["add"]) From c8a370d3877ff3d875ebe66710056d058175caf3 Mon Sep 17 00:00:00 2001 From: junderw Date: Wed, 26 Jul 2023 00:16:22 -0700 Subject: [PATCH 6/8] Fix order of operations to not fail --- src/new_index/mempool.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index b896cb1ba..f393eb2bd 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -396,14 +396,21 @@ impl Mempool { } debug!("Adding {} transactions to Mempool", txlen); - // Phase 1: index history and spend edges (some txos can be missing) - let txids: Vec<_> = txs.iter().map(Transaction::txid).collect(); + let mut txids = Vec::with_capacity(txs.len()); + // Phase 1: add to txstore + for tx in txs { + let txid = tx.txid(); + txids.push(txid); + self.txstore.insert(txid, tx); + } + + // Phase 2: index history and spend edges (some txos can be missing) let txos = self.lookup_txos(&self.get_prevouts(&txids)); // Count how many transactions were actually processed. let mut processed_count = 0; - // Phase 2: Iterate over the transactions and do the following: + // Phase 3: Iterate over the transactions and do the following: // 1. Find all of the TxOuts of each input parent using `txos` // 2. If any parent wasn't found, skip parsing this transaction // 3. Insert TxFeeInfo into info. @@ -412,14 +419,8 @@ impl Mempool { // 6. Insert all TxHistory into history. // 7. Insert the tx edges into edges (HashMap of (Outpoint, (Txid, vin))) // 8. (Liquid only) Parse assets of tx. - for owned_tx in txs { - let txid = owned_tx.txid(); - - let entry = self.txstore.entry(txid); - // Note: This fn doesn't overwrite existing transactions, - // But that's ok, we didn't insert unrelated txes - // into any given txid anyways, so this will always insert. - let tx = &*entry.or_insert_with(|| owned_tx); + for txid in txids { + let tx = self.txstore.get(&txid).expect("missing tx from txstore"); let prevouts = match extract_tx_prevouts(tx, &txos, false) { Ok(v) => v, From 3188ddf7d4f160219aa312108328952d16a6696f Mon Sep 17 00:00:00 2001 From: junderw Date: Wed, 26 Jul 2023 01:21:40 -0700 Subject: [PATCH 7/8] Fix /tx/HASH bug --- src/rest.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rest.rs b/src/rest.rs index bbc14ae68..920d90aa2 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -981,9 +981,8 @@ fn handle_request( let ttl = ttl_by_depth(blockid.as_ref().map(|b| b.height), query); let mut tx = prepare_txs(vec![(tx, blockid)], query, config); - tx.remove(0); - json_response(tx, ttl) + json_response(tx.remove(0), ttl) } (&Method::GET, Some(&"tx"), Some(hash), Some(out_type @ &"hex"), None, None) | (&Method::GET, Some(&"tx"), Some(hash), Some(out_type @ &"raw"), None, None) => { From f0e1db32dcf4eed2aafe6412a37b604cfc1d5eba Mon Sep 17 00:00:00 2001 From: junderw Date: Sat, 29 Jul 2023 10:40:00 -0700 Subject: [PATCH 8/8] Feat: Use f64 for difficulty. Porting Bitcoin Core algorithm. JSON representation varies slightly. --- src/rest.rs | 155 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 150 insertions(+), 5 deletions(-) diff --git a/src/rest.rs b/src/rest.rs index 920d90aa2..9e92246e2 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -69,7 +69,7 @@ struct BlockValue { #[cfg(not(feature = "liquid"))] bits: u32, #[cfg(not(feature = "liquid"))] - difficulty: u64, + difficulty: f64, #[cfg(feature = "liquid")] #[serde(skip_serializing_if = "Option::is_none")] @@ -78,7 +78,7 @@ struct BlockValue { impl BlockValue { #[cfg_attr(feature = "liquid", allow(unused_variables))] - fn new(blockhm: BlockHeaderMeta, network: Network) -> Self { + fn new(blockhm: BlockHeaderMeta) -> Self { let header = blockhm.header_entry.header(); BlockValue { id: header.block_hash().to_hex(), @@ -106,7 +106,7 @@ impl BlockValue { #[cfg(not(feature = "liquid"))] nonce: header.nonce, #[cfg(not(feature = "liquid"))] - difficulty: header.difficulty(bitcoin::Network::from(network)), + difficulty: difficulty_new(header), #[cfg(feature = "liquid")] ext: Some(json!(header.ext)), @@ -114,6 +114,27 @@ impl BlockValue { } } +/// Calculate the difficulty of a BlockHeader +/// using Bitcoin Core code ported to Rust. +/// +/// https://github.com/bitcoin/bitcoin/blob/v25.0/src/rpc/blockchain.cpp#L75-L97 +#[cfg_attr(feature = "liquid", allow(dead_code))] +fn difficulty_new(bh: &bitcoin::BlockHeader) -> f64 { + let mut n_shift = bh.bits >> 24 & 0xff; + let mut d_diff = (0x0000ffff as f64) / ((bh.bits & 0x00ffffff) as f64); + + while n_shift < 29 { + d_diff *= 256.0; + n_shift += 1; + } + while n_shift > 29 { + d_diff /= 256.0; + n_shift -= 1; + } + + d_diff +} + #[derive(Serialize, Deserialize)] struct TransactionValue { txid: Txid, @@ -681,7 +702,7 @@ fn handle_request( .chain() .get_block_with_meta(&hash) .ok_or_else(|| HttpError::not_found("Block not found".to_string()))?; - let block_value = BlockValue::new(blockhm, config.network_type); + let block_value = BlockValue::new(blockhm); json_response(block_value, TTL_LONG) } (&Method::GET, Some(&"block"), Some(hash), Some(&"status"), None, None) => { @@ -1325,7 +1346,7 @@ fn blocks( current_hash = blockhm.header_entry.header().prev_blockhash; #[allow(unused_mut)] - let mut value = BlockValue::new(blockhm, config.network_type); + let mut value = BlockValue::new(blockhm); #[cfg(feature = "liquid")] { @@ -1533,4 +1554,128 @@ mod tests { assert!(err.is_err()); } + + #[test] + fn test_difficulty_new() { + use super::difficulty_new; + + let vectors = [ + ( + // bits in header + 0x17053894, + // expected output (Rust) + 53911173001054.586, + // Block hash where found (for getblockheader) + "0000000000000000000050b050758dd2ccb0ba96ad5e95db84efd2f6c05e4e90", + // difficulty returned by Bitcoin Core v25 + "53911173001054.59", + ), + ( + 0x1a0c2a12, + 1379192.2882280778, + "0000000000000bc7636ffbc1cf90cf4a2674de7fcadbc6c9b63d31f07cb3c2c2", + "1379192.288228078", + ), + ( + 0x19262222, + 112628548.66634709, + "000000000000000996b1f06771a81bcf7b15c5f859b6f8329016f01b0442ca72", + "112628548.6663471", + ), + ( + 0x1d00c428, + 1.3050621315915245, + "0000000034014d731a3e1ad6078662ce19b08179dcc7ec0f5f717d4b58060736", + "1.305062131591525", + ), + ( + 0, + f64::INFINITY, + "[No Blockhash]", + "[No Core difficulty, just checking edge cases]", + ), + ( + 0x00000001, + 4.523059468369196e74, + "[No Blockhash]", + "[No Core difficulty, just checking edge cases]", + ), + ( + 0x1d00ffff, + 1.0, + "[No Blockhash]", + "[No Core difficulty, just checking MAX_TARGET]", + ), + ( + 0x1c7fff80, + 2.0, + "[No Blockhash]", + "[No Core difficulty, just checking MAX_TARGET >> 1]", + ), + ( + 0x1b00ffff, + 65536.0, + "[No Blockhash]", + "[No Core difficulty, just checking MAX_TARGET >> 16]", + ), + ( + 0x1a7fff80, + 131072.0, + "[No Blockhash]", + "[No Core difficulty, just checking MAX_TARGET >> 17]", + ), + ( + 0x1d01fffe, + 0.5, + "[No Blockhash]", + "[No Core difficulty, just checking MAX_TARGET << 1]", + ), + ( + 0x1f000080, + 0.007812380790710449, + "[No Blockhash]", + "[No Core difficulty, just checking 2**255]", + ), + ( + 0x1e00ffff, + 0.00390625, // 2.0**-8 + "[No Blockhash]", + "[No Core difficulty, just checking MAX_TARGET << 8]", + ), + ( + 0x1e00ff00, + 0.0039215087890625, + "[No Blockhash]", + "[No Core difficulty, just checking MAX_TARGET << 8 - two `f` chars]", + ), + ( + 0x1f0000ff, + 0.0039215087890625, + "[No Blockhash]", + "[No Core difficulty, just checking MAX_TARGET << 8]", + ), + ]; + + let to_bh = |b| bitcoin::BlockHeader { + version: 1, + prev_blockhash: "0000000000000000000000000000000000000000000000000000000000000000" + .parse() + .unwrap(), + merkle_root: "0000000000000000000000000000000000000000000000000000000000000000" + .parse() + .unwrap(), + time: 0, + bits: b, + nonce: 0, + }; + + for (bits, expected, hash, core_difficulty) in vectors { + let result = difficulty_new(&to_bh(bits)); + assert_eq!( + result, expected, + "Block {} difficulty is {} but Core difficulty is {}", + hash, result, core_difficulty, + ); + } + } }