Merge remote-tracking branch 'mempool/mempool' into mononaut/wallet-apis

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mempool-electrs"
-version = "3.0.0-dev"
+version = "3.1.0-dev"
 authors = [
     "Roman Zeyde <[email protected]>",
     "Nadav Ivgi <[email protected]>",

rust-toolchain

@@ -1 +1 @@
-1.70
+1.80

src/bin/electrs.rs

@@ -50,6 +50,7 @@ fn run_server(config: Arc<Config>) -> Result<()> {
         config.daemon_rpc_addr,
         config.cookie_getter(),
         config.network_type,
+        config.magic,
         signal.clone(),
         &metrics,
     )?);
@@ -74,7 +75,18 @@ fn run_server(config: Arc<Config>) -> Result<()> {
         &metrics,
         Arc::clone(&config),
     )));
-    mempool.write().unwrap().update(&daemon)?;
+    loop {
+        match Mempool::update(&mempool, &daemon) {
+            Ok(_) => break,
+            Err(e) => {
+                warn!(
+                    "Error performing initial mempool update, trying again in 5 seconds: {}",
+                    e.display_chain()
+                );
+                signal.wait(Duration::from_secs(5), false)?;
+            }
+        }
+    }
 
     #[cfg(feature = "liquid")]
     let asset_db = config.asset_db_path.as_ref().map(|db_dir| {
@@ -136,7 +148,13 @@ fn run_server(config: Arc<Config>) -> Result<()> {
         };
 
         // Update mempool
-        mempool.write().unwrap().update(&daemon)?;
+        if let Err(e) = Mempool::update(&mempool, &daemon) {
+            // Log the error if the result is an Err
+            warn!(
+                "Error updating mempool, skipping mempool update: {}",
+                e.display_chain()
+            );
+        }
 
         // Update subscribed clients
         electrum_server.notify();

src/bin/popular-scripts.rs

@@ -95,8 +95,7 @@ fn run_iterator(
         "Thread ({thread_id:?}) Seeking DB to beginning of tx histories for b'H' + {}",
         hex::encode([first_byte])
     );
-    // H = 72
-    let mut compare_vec: Vec<u8> = vec![72, first_byte];
+    let mut compare_vec: Vec<u8> = vec![b'H', first_byte];
     iter.seek(&compare_vec); // Seek to beginning of our section
 
     // Insert the byte of the next section for comparing
@@ -122,7 +121,7 @@ fn run_iterator(
     while iter.valid() {
         let key = iter.key().unwrap();
 
-        if is_finished(key) {
+        if key.is_empty() || key[0] != b'H' || is_finished(key) {
             // We have left the txhistory section,
             // but we need to check the final scripthash
             send_if_popular(

src/bin/tx-fingerprint-stats.rs

@@ -35,6 +35,7 @@ fn main() {
             config.daemon_rpc_addr,
             config.cookie_getter(),
             config.network_type,
+            config.magic,
             signal,
             &metrics,
         )

src/chain.rs

@@ -1,3 +1,5 @@
+use std::str::FromStr;
+
 #[cfg(not(feature = "liquid"))] // use regular Bitcoin data structures
 pub use bitcoin::{
     blockdata::{opcodes, script, witness::Witness},
@@ -32,6 +34,8 @@ pub enum Network {
     #[cfg(not(feature = "liquid"))]
     Testnet,
     #[cfg(not(feature = "liquid"))]
+    Testnet4,
+    #[cfg(not(feature = "liquid"))]
     Regtest,
     #[cfg(not(feature = "liquid"))]
     Signet,
@@ -124,27 +128,39 @@ impl Network {
 
 pub fn genesis_hash(network: Network) -> BlockHash {
     #[cfg(not(feature = "liquid"))]
-    return bitcoin_genesis_hash(network.into());
+    return bitcoin_genesis_hash(network);
     #[cfg(feature = "liquid")]
     return liquid_genesis_hash(network);
 }
 
-pub fn bitcoin_genesis_hash(network: BNetwork) -> bitcoin::BlockHash {
+pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash {
     lazy_static! {
         static ref BITCOIN_GENESIS: bitcoin::BlockHash =
             genesis_block(BNetwork::Bitcoin).block_hash();
         static ref TESTNET_GENESIS: bitcoin::BlockHash =
             genesis_block(BNetwork::Testnet).block_hash();
+        static ref TESTNET4_GENESIS: bitcoin::BlockHash = bitcoin::BlockHash::from_str(
+            "00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043"
+        )
+        .unwrap();
         static ref REGTEST_GENESIS: bitcoin::BlockHash =
             genesis_block(BNetwork::Regtest).block_hash();
         static ref SIGNET_GENESIS: bitcoin::BlockHash =
             genesis_block(BNetwork::Signet).block_hash();
     }
+    #[cfg(not(feature = "liquid"))]
     match network {
-        BNetwork::Bitcoin => *BITCOIN_GENESIS,
-        BNetwork::Testnet => *TESTNET_GENESIS,
-        BNetwork::Regtest => *REGTEST_GENESIS,
-        BNetwork::Signet => *SIGNET_GENESIS,
+        Network::Bitcoin => *BITCOIN_GENESIS,
+        Network::Testnet => *TESTNET_GENESIS,
+        Network::Testnet4 => *TESTNET4_GENESIS,
+        Network::Regtest => *REGTEST_GENESIS,
+        Network::Signet => *SIGNET_GENESIS,
+    }
+    #[cfg(feature = "liquid")]
+    match network {
+        Network::Liquid => *BITCOIN_GENESIS,
+        Network::LiquidTestnet => *TESTNET_GENESIS,
+        Network::LiquidRegtest => *REGTEST_GENESIS,
     }
 }
 
@@ -174,6 +190,8 @@ impl From<&str> for Network {
             #[cfg(not(feature = "liquid"))]
             "testnet" => Network::Testnet,
             #[cfg(not(feature = "liquid"))]
+            "testnet4" => Network::Testnet4,
+            #[cfg(not(feature = "liquid"))]
             "regtest" => Network::Regtest,
             #[cfg(not(feature = "liquid"))]
             "signet" => Network::Signet,
@@ -196,6 +214,7 @@ impl From<Network> for BNetwork {
         match network {
             Network::Bitcoin => BNetwork::Bitcoin,
             Network::Testnet => BNetwork::Testnet,
+            Network::Testnet4 => BNetwork::Testnet,
             Network::Regtest => BNetwork::Regtest,
             Network::Signet => BNetwork::Signet,
         }

src/config.rs

@@ -33,6 +33,7 @@ pub struct Config {
     // See below for the documentation of each field:
     pub log: stderrlog::StdErrLog,
     pub network_type: Network,
+    pub magic: Option<u32>,
     pub db_path: PathBuf,
     pub daemon_dir: PathBuf,
     pub blocks_dir: PathBuf,
@@ -137,6 +138,12 @@ impl Config {
                     .help(&network_help)
                     .takes_value(true),
             )
+            .arg(
+                Arg::with_name("magic")
+                    .long("magic")
+                    .default_value("")
+                    .takes_value(true),
+            )
             .arg(
                 Arg::with_name("electrum_rpc_addr")
                     .long("electrum-rpc-addr")
@@ -328,6 +335,10 @@ impl Config {
 
         let network_name = m.value_of("network").unwrap_or("mainnet");
         let network_type = Network::from(network_name);
+        let magic: Option<u32> = m
+            .value_of("magic")
+            .filter(|s| !s.is_empty())
+            .map(|s| u32::from_str_radix(s, 16).expect("invalid network magic"));
         let db_dir = Path::new(m.value_of("db_dir").unwrap_or("./db"));
         let db_path = db_dir.join(network_name);
 
@@ -353,6 +364,8 @@ impl Config {
             Network::Regtest => 18443,
             #[cfg(not(feature = "liquid"))]
             Network::Signet => 38332,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 48332,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 7041,
@@ -365,6 +378,8 @@ impl Config {
             #[cfg(not(feature = "liquid"))]
             Network::Testnet => 60001,
             #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 40001,
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 60401,
             #[cfg(not(feature = "liquid"))]
             Network::Signet => 60601,
@@ -385,6 +400,8 @@ impl Config {
             Network::Regtest => 3002,
             #[cfg(not(feature = "liquid"))]
             Network::Signet => 3003,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 3004,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 3000,
@@ -401,6 +418,8 @@ impl Config {
             #[cfg(not(feature = "liquid"))]
             Network::Regtest => 24224,
             #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 44224,
+            #[cfg(not(feature = "liquid"))]
             Network::Signet => 54224,
 
             #[cfg(feature = "liquid")]
@@ -449,6 +468,8 @@ impl Config {
             #[cfg(not(feature = "liquid"))]
             Network::Testnet => daemon_dir.push("testnet3"),
             #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => daemon_dir.push("testnet4"),
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => daemon_dir.push("regtest"),
             #[cfg(not(feature = "liquid"))]
             Network::Signet => daemon_dir.push("signet"),
@@ -486,6 +507,7 @@ impl Config {
         let config = Config {
             log,
             network_type,
+            magic,
             db_path,
             daemon_dir,
             blocks_dir,

src/daemon.rs

@@ -117,6 +117,26 @@ struct NetworkInfo {
     relayfee: f64, // in BTC/kB
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+struct MempoolFees {
+    base: f64,
+    #[serde(rename = "effective-feerate")]
+    effective_feerate: f64,
+    #[serde(rename = "effective-includes")]
+    effective_includes: Vec<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct MempoolAcceptResult {
+    txid: String,
+    wtxid: String,
+    allowed: Option<bool>,
+    vsize: Option<u32>,
+    fees: Option<MempoolFees>,
+    #[serde(rename = "reject-reason")]
+    reject_reason: Option<String>,
+}
+
 pub trait CookieGetter: Send + Sync {
     fn get(&self) -> Result<Vec<u8>>;
 }
@@ -264,6 +284,7 @@ pub struct Daemon {
     daemon_dir: PathBuf,
     blocks_dir: PathBuf,
     network: Network,
+    magic: Option<u32>,
     conn: Mutex<Connection>,
     message_id: Counter, // for monotonic JSONRPC 'id'
     signal: Waiter,
@@ -274,19 +295,22 @@ pub struct Daemon {
 }
 
 impl Daemon {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         daemon_dir: PathBuf,
         blocks_dir: PathBuf,
         daemon_rpc_addr: SocketAddr,
         cookie_getter: Arc<dyn CookieGetter>,
         network: Network,
+        magic: Option<u32>,
         signal: Waiter,
         metrics: &Metrics,
     ) -> Result<Daemon> {
         let daemon = Daemon {
             daemon_dir,
             blocks_dir,
             network,
+            magic,
             conn: Mutex::new(Connection::new(
                 daemon_rpc_addr,
                 cookie_getter,
@@ -321,10 +345,10 @@ impl Daemon {
             let mempool = daemon.getmempoolinfo()?;
 
             let ibd_done = if network.is_regtest() {
-                info.blocks == 0 && info.headers == 0
+                info.blocks == info.headers
             } else {
-                false
-            } || !info.initialblockdownload.unwrap_or(false);
+                !info.initialblockdownload.unwrap_or(false)
+            };
 
             if mempool.loaded && ibd_done && info.blocks == info.headers {
                 break;
@@ -347,6 +371,7 @@ impl Daemon {
             daemon_dir: self.daemon_dir.clone(),
             blocks_dir: self.blocks_dir.clone(),
             network: self.network,
+            magic: self.magic,
             conn: Mutex::new(self.conn.lock().unwrap().reconnect()?),
             message_id: Counter::new(),
             signal: self.signal.clone(),
@@ -367,7 +392,7 @@ impl Daemon {
     }
 
     pub fn magic(&self) -> u32 {
-        self.network.magic()
+        self.magic.unwrap_or_else(|| self.network.magic())
     }
 
     fn call_jsonrpc(&self, method: &str, request: &Value) -> Result<Value> {
@@ -582,6 +607,20 @@ impl Daemon {
             .chain_err(|| "failed to parse txid")
     }
 
+    pub fn test_mempool_accept(
+        &self,
+        txhex: Vec<String>,
+        maxfeerate: Option<f64>,
+    ) -> Result<Vec<MempoolAcceptResult>> {
+        let params = match maxfeerate {
+            Some(rate) => json!([txhex, format!("{:.8}", rate)]),
+            None => json!([txhex]),
+        };
+        let result = self.request("testmempoolaccept", params)?;
+        serde_json::from_value::<Vec<MempoolAcceptResult>>(result)
+            .chain_err(|| "invalid testmempoolaccept reply")
+    }
+
     // Get estimated feerates for the provided confirmation targets using a batch RPC request
     // Missing estimates are logged but do not cause a failure, whatever is available is returned
     #[allow(clippy::float_cmp)]

src/electrum/client.rs

@@ -3,7 +3,6 @@ use std::convert::TryFrom;
 
 use bitcoin::hashes::Hash;
 pub use electrum_client::client::Client;
-pub use electrum_client::Error as ElectrumError;
 pub use electrum_client::ServerFeaturesRes;
 
 use crate::chain::BlockHash;