diff --git a/Cargo.lock b/Cargo.lock index db6d368a..1243f4ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -822,7 +822,7 @@ dependencies = [ [[package]] name = "mempool-electrs" -version = "3.0.0-dev" +version = "3.1.0-dev" dependencies = [ "arrayref", "base64 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index 58c5b579..f41a588a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mempool-electrs" -version = "3.0.0-dev" +version = "3.1.0-dev" authors = [ "Roman Zeyde ", "Nadav Ivgi ", diff --git a/rust-toolchain b/rust-toolchain index bfe79d0b..d456f745 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.70 +1.80 diff --git a/src/bin/electrs.rs b/src/bin/electrs.rs index 91a51eef..0c649355 100644 --- a/src/bin/electrs.rs +++ b/src/bin/electrs.rs @@ -50,6 +50,7 @@ fn run_server(config: Arc) -> Result<()> { config.daemon_rpc_addr, config.cookie_getter(), config.network_type, + config.magic, signal.clone(), &metrics, )?); @@ -74,7 +75,18 @@ fn run_server(config: Arc) -> Result<()> { &metrics, Arc::clone(&config), ))); - mempool.write().unwrap().update(&daemon)?; + loop { + match Mempool::update(&mempool, &daemon) { + Ok(_) => break, + Err(e) => { + warn!( + "Error performing initial mempool update, trying again in 5 seconds: {}", + e.display_chain() + ); + signal.wait(Duration::from_secs(5), false)?; + } + } + } #[cfg(feature = "liquid")] let asset_db = config.asset_db_path.as_ref().map(|db_dir| { @@ -136,7 +148,13 @@ fn run_server(config: Arc) -> Result<()> { }; // Update mempool - mempool.write().unwrap().update(&daemon)?; + if let Err(e) = Mempool::update(&mempool, &daemon) { + // Log the error if the result is an Err + warn!( + "Error updating mempool, skipping mempool update: {}", + e.display_chain() + ); + } // Update subscribed clients electrum_server.notify(); diff --git a/src/bin/popular-scripts.rs b/src/bin/popular-scripts.rs index 9005f5b8..db928d06 100644 --- a/src/bin/popular-scripts.rs +++ b/src/bin/popular-scripts.rs @@ -95,8 +95,7 @@ fn run_iterator( "Thread ({thread_id:?}) Seeking DB to beginning of tx histories for b'H' + {}", hex::encode([first_byte]) ); - // H = 72 - let mut compare_vec: Vec = vec![72, first_byte]; + let mut compare_vec: Vec = vec![b'H', first_byte]; iter.seek(&compare_vec); // Seek to beginning of our section // Insert the byte of the next section for comparing @@ -122,7 +121,7 @@ fn run_iterator( while iter.valid() { let key = iter.key().unwrap(); - if is_finished(key) { + if key.is_empty() || key[0] != b'H' || is_finished(key) { // We have left the txhistory section, // but we need to check the final scripthash send_if_popular( diff --git a/src/bin/tx-fingerprint-stats.rs b/src/bin/tx-fingerprint-stats.rs index 55cb6797..5b38561f 100644 --- a/src/bin/tx-fingerprint-stats.rs +++ b/src/bin/tx-fingerprint-stats.rs @@ -35,6 +35,7 @@ fn main() { config.daemon_rpc_addr, config.cookie_getter(), config.network_type, + config.magic, signal, &metrics, ) diff --git a/src/chain.rs b/src/chain.rs index de726186..ccb2b353 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -1,3 +1,5 @@ +use std::str::FromStr; + #[cfg(not(feature = "liquid"))] // use regular Bitcoin data structures pub use bitcoin::{ blockdata::{opcodes, script, witness::Witness}, @@ -32,6 +34,8 @@ pub enum Network { #[cfg(not(feature = "liquid"))] Testnet, #[cfg(not(feature = "liquid"))] + Testnet4, + #[cfg(not(feature = "liquid"))] Regtest, #[cfg(not(feature = "liquid"))] Signet, @@ -124,27 +128,39 @@ impl Network { pub fn genesis_hash(network: Network) -> BlockHash { #[cfg(not(feature = "liquid"))] - return bitcoin_genesis_hash(network.into()); + return bitcoin_genesis_hash(network); #[cfg(feature = "liquid")] return liquid_genesis_hash(network); } -pub fn bitcoin_genesis_hash(network: BNetwork) -> bitcoin::BlockHash { +pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash { lazy_static! { static ref BITCOIN_GENESIS: bitcoin::BlockHash = genesis_block(BNetwork::Bitcoin).block_hash(); static ref TESTNET_GENESIS: bitcoin::BlockHash = genesis_block(BNetwork::Testnet).block_hash(); + static ref TESTNET4_GENESIS: bitcoin::BlockHash = bitcoin::BlockHash::from_str( + "00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043" + ) + .unwrap(); static ref REGTEST_GENESIS: bitcoin::BlockHash = genesis_block(BNetwork::Regtest).block_hash(); static ref SIGNET_GENESIS: bitcoin::BlockHash = genesis_block(BNetwork::Signet).block_hash(); } + #[cfg(not(feature = "liquid"))] match network { - BNetwork::Bitcoin => *BITCOIN_GENESIS, - BNetwork::Testnet => *TESTNET_GENESIS, - BNetwork::Regtest => *REGTEST_GENESIS, - BNetwork::Signet => *SIGNET_GENESIS, + Network::Bitcoin => *BITCOIN_GENESIS, + Network::Testnet => *TESTNET_GENESIS, + Network::Testnet4 => *TESTNET4_GENESIS, + Network::Regtest => *REGTEST_GENESIS, + Network::Signet => *SIGNET_GENESIS, + } + #[cfg(feature = "liquid")] + match network { + Network::Liquid => *BITCOIN_GENESIS, + Network::LiquidTestnet => *TESTNET_GENESIS, + Network::LiquidRegtest => *REGTEST_GENESIS, } } @@ -174,6 +190,8 @@ impl From<&str> for Network { #[cfg(not(feature = "liquid"))] "testnet" => Network::Testnet, #[cfg(not(feature = "liquid"))] + "testnet4" => Network::Testnet4, + #[cfg(not(feature = "liquid"))] "regtest" => Network::Regtest, #[cfg(not(feature = "liquid"))] "signet" => Network::Signet, @@ -196,6 +214,7 @@ impl From for BNetwork { match network { Network::Bitcoin => BNetwork::Bitcoin, Network::Testnet => BNetwork::Testnet, + Network::Testnet4 => BNetwork::Testnet, Network::Regtest => BNetwork::Regtest, Network::Signet => BNetwork::Signet, } diff --git a/src/config.rs b/src/config.rs index 8278d985..a5e903ce 100644 --- a/src/config.rs +++ b/src/config.rs @@ -33,6 +33,7 @@ pub struct Config { // See below for the documentation of each field: pub log: stderrlog::StdErrLog, pub network_type: Network, + pub magic: Option, pub db_path: PathBuf, pub daemon_dir: PathBuf, pub blocks_dir: PathBuf, @@ -137,6 +138,12 @@ impl Config { .help(&network_help) .takes_value(true), ) + .arg( + Arg::with_name("magic") + .long("magic") + .default_value("") + .takes_value(true), + ) .arg( Arg::with_name("electrum_rpc_addr") .long("electrum-rpc-addr") @@ -328,6 +335,10 @@ impl Config { let network_name = m.value_of("network").unwrap_or("mainnet"); let network_type = Network::from(network_name); + let magic: Option = m + .value_of("magic") + .filter(|s| !s.is_empty()) + .map(|s| u32::from_str_radix(s, 16).expect("invalid network magic")); let db_dir = Path::new(m.value_of("db_dir").unwrap_or("./db")); let db_path = db_dir.join(network_name); @@ -353,6 +364,8 @@ impl Config { Network::Regtest => 18443, #[cfg(not(feature = "liquid"))] Network::Signet => 38332, + #[cfg(not(feature = "liquid"))] + Network::Testnet4 => 48332, #[cfg(feature = "liquid")] Network::Liquid => 7041, @@ -365,6 +378,8 @@ impl Config { #[cfg(not(feature = "liquid"))] Network::Testnet => 60001, #[cfg(not(feature = "liquid"))] + Network::Testnet4 => 40001, + #[cfg(not(feature = "liquid"))] Network::Regtest => 60401, #[cfg(not(feature = "liquid"))] Network::Signet => 60601, @@ -385,6 +400,8 @@ impl Config { Network::Regtest => 3002, #[cfg(not(feature = "liquid"))] Network::Signet => 3003, + #[cfg(not(feature = "liquid"))] + Network::Testnet4 => 3004, #[cfg(feature = "liquid")] Network::Liquid => 3000, @@ -401,6 +418,8 @@ impl Config { #[cfg(not(feature = "liquid"))] Network::Regtest => 24224, #[cfg(not(feature = "liquid"))] + Network::Testnet4 => 44224, + #[cfg(not(feature = "liquid"))] Network::Signet => 54224, #[cfg(feature = "liquid")] @@ -449,6 +468,8 @@ impl Config { #[cfg(not(feature = "liquid"))] Network::Testnet => daemon_dir.push("testnet3"), #[cfg(not(feature = "liquid"))] + Network::Testnet4 => daemon_dir.push("testnet4"), + #[cfg(not(feature = "liquid"))] Network::Regtest => daemon_dir.push("regtest"), #[cfg(not(feature = "liquid"))] Network::Signet => daemon_dir.push("signet"), @@ -486,6 +507,7 @@ impl Config { let config = Config { log, network_type, + magic, db_path, daemon_dir, blocks_dir, diff --git a/src/daemon.rs b/src/daemon.rs index b794a1f9..f04045d0 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -117,6 +117,26 @@ struct NetworkInfo { relayfee: f64, // in BTC/kB } +#[derive(Serialize, Deserialize, Debug)] +struct MempoolFees { + base: f64, + #[serde(rename = "effective-feerate")] + effective_feerate: f64, + #[serde(rename = "effective-includes")] + effective_includes: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct MempoolAcceptResult { + txid: String, + wtxid: String, + allowed: Option, + vsize: Option, + fees: Option, + #[serde(rename = "reject-reason")] + reject_reason: Option, +} + pub trait CookieGetter: Send + Sync { fn get(&self) -> Result>; } @@ -264,6 +284,7 @@ pub struct Daemon { daemon_dir: PathBuf, blocks_dir: PathBuf, network: Network, + magic: Option, conn: Mutex, message_id: Counter, // for monotonic JSONRPC 'id' signal: Waiter, @@ -274,12 +295,14 @@ pub struct Daemon { } impl Daemon { + #[allow(clippy::too_many_arguments)] pub fn new( daemon_dir: PathBuf, blocks_dir: PathBuf, daemon_rpc_addr: SocketAddr, cookie_getter: Arc, network: Network, + magic: Option, signal: Waiter, metrics: &Metrics, ) -> Result { @@ -287,6 +310,7 @@ impl Daemon { daemon_dir, blocks_dir, network, + magic, conn: Mutex::new(Connection::new( daemon_rpc_addr, cookie_getter, @@ -321,10 +345,10 @@ impl Daemon { let mempool = daemon.getmempoolinfo()?; let ibd_done = if network.is_regtest() { - info.blocks == 0 && info.headers == 0 + info.blocks == info.headers } else { - false - } || !info.initialblockdownload.unwrap_or(false); + !info.initialblockdownload.unwrap_or(false) + }; if mempool.loaded && ibd_done && info.blocks == info.headers { break; @@ -347,6 +371,7 @@ impl Daemon { daemon_dir: self.daemon_dir.clone(), blocks_dir: self.blocks_dir.clone(), network: self.network, + magic: self.magic, conn: Mutex::new(self.conn.lock().unwrap().reconnect()?), message_id: Counter::new(), signal: self.signal.clone(), @@ -367,7 +392,7 @@ impl Daemon { } pub fn magic(&self) -> u32 { - self.network.magic() + self.magic.unwrap_or_else(|| self.network.magic()) } fn call_jsonrpc(&self, method: &str, request: &Value) -> Result { @@ -582,6 +607,20 @@ impl Daemon { .chain_err(|| "failed to parse txid") } + pub fn test_mempool_accept( + &self, + txhex: Vec, + maxfeerate: Option, + ) -> Result> { + let params = match maxfeerate { + Some(rate) => json!([txhex, format!("{:.8}", rate)]), + None => json!([txhex]), + }; + let result = self.request("testmempoolaccept", params)?; + serde_json::from_value::>(result) + .chain_err(|| "invalid testmempoolaccept reply") + } + // Get estimated feerates for the provided confirmation targets using a batch RPC request // Missing estimates are logged but do not cause a failure, whatever is available is returned #[allow(clippy::float_cmp)] diff --git a/src/electrum/client.rs b/src/electrum/client.rs index 04d6ffba..02cc2d15 100644 --- a/src/electrum/client.rs +++ b/src/electrum/client.rs @@ -3,7 +3,6 @@ use std::convert::TryFrom; use bitcoin::hashes::Hash; pub use electrum_client::client::Client; -pub use electrum_client::Error as ElectrumError; pub use electrum_client::ServerFeaturesRes; use crate::chain::BlockHash; diff --git a/src/electrum/server.rs b/src/electrum/server.rs index ae427cd2..d1302b23 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -189,7 +189,7 @@ impl Connection { .chain_err(|| "discovery is disabled")?; let features = params - .get(0) + .first() .chain_err(|| "missing features param")? .clone(); let features = serde_json::from_value(features).chain_err(|| "invalid features")?; @@ -203,7 +203,7 @@ impl Connection { } fn blockchain_block_header(&self, params: &[Value]) -> Result { - let height = usize_from_value(params.get(0), "height")?; + let height = usize_from_value(params.first(), "height")?; let cp_height = usize_from_value_or(params.get(1), "cp_height", 0)?; let raw_header_hex: String = self @@ -226,7 +226,7 @@ impl Connection { } fn blockchain_block_headers(&self, params: &[Value]) -> Result { - let start_height = usize_from_value(params.get(0), "start_height")?; + let start_height = usize_from_value(params.first(), "start_height")?; let count = MAX_HEADERS.min(usize_from_value(params.get(1), "count")?); let cp_height = usize_from_value_or(params.get(2), "cp_height", 0)?; let heights: Vec = (start_height..(start_height + count)).collect(); @@ -261,7 +261,7 @@ impl Connection { } fn blockchain_estimatefee(&self, params: &[Value]) -> Result { - let conf_target = usize_from_value(params.get(0), "blocks_count")?; + let conf_target = usize_from_value(params.first(), "blocks_count")?; let fee_rate = self .query .estimate_fee(conf_target as u16) @@ -277,7 +277,7 @@ impl Connection { } fn blockchain_scripthash_subscribe(&mut self, params: &[Value]) -> Result { - let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?; + let script_hash = hash_from_value(params.first()).chain_err(|| "bad script_hash")?; let history_txids = get_history(&self.query, &script_hash[..], self.txs_limit)?; let status_hash = get_status_hash(history_txids, &self.query) @@ -295,7 +295,7 @@ impl Connection { #[cfg(not(feature = "liquid"))] fn blockchain_scripthash_get_balance(&self, params: &[Value]) -> Result { - let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?; + let script_hash = hash_from_value(params.first()).chain_err(|| "bad script_hash")?; let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..]); Ok(json!({ @@ -305,7 +305,7 @@ impl Connection { } fn blockchain_scripthash_get_history(&self, params: &[Value]) -> Result { - let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?; + let script_hash = hash_from_value(params.first()).chain_err(|| "bad script_hash")?; let history_txids = get_history(&self.query, &script_hash[..], self.txs_limit)?; Ok(json!(history_txids @@ -323,7 +323,7 @@ impl Connection { } fn blockchain_scripthash_listunspent(&self, params: &[Value]) -> Result { - let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?; + let script_hash = hash_from_value(params.first()).chain_err(|| "bad script_hash")?; let utxos = self.query.utxo(&script_hash[..])?; let to_json = |utxo: Utxo| { @@ -351,7 +351,7 @@ impl Connection { } fn blockchain_transaction_broadcast(&self, params: &[Value]) -> Result { - let tx = params.get(0).chain_err(|| "missing tx")?; + let tx = params.first().chain_err(|| "missing tx")?; let tx = tx.as_str().chain_err(|| "non-string tx")?.to_string(); let txid = self.query.broadcast_raw(&tx)?; if let Err(e) = self.chan.sender().try_send(Message::PeriodicUpdate) { @@ -361,7 +361,7 @@ impl Connection { } fn blockchain_transaction_get(&self, params: &[Value]) -> Result { - let tx_hash = Txid::from(hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?); + let tx_hash = Txid::from(hash_from_value(params.first()).chain_err(|| "bad tx_hash")?); let verbose = match params.get(1) { Some(value) => value.as_bool().chain_err(|| "non-bool verbose value")?, None => false, @@ -380,7 +380,7 @@ impl Connection { } fn blockchain_transaction_get_merkle(&self, params: &[Value]) -> Result { - let txid = Txid::from(hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?); + let txid = Txid::from(hash_from_value(params.first()).chain_err(|| "bad tx_hash")?); let height = usize_from_value(params.get(1), "height")?; let blockid = self .query @@ -399,7 +399,7 @@ impl Connection { } fn blockchain_transaction_id_from_pos(&self, params: &[Value]) -> Result { - let height = usize_from_value(params.get(0), "height")?; + let height = usize_from_value(params.first(), "height")?; let tx_pos = usize_from_value(params.get(1), "tx_pos")?; let want_merkle = bool_from_value_or(params.get(2), "merkle", false)?; @@ -513,7 +513,6 @@ impl Connection { } fn handle_replies(&mut self, shutdown: crossbeam_channel::Receiver<()>) -> Result<()> { - let empty_params = json!([]); loop { crossbeam_channel::select! { recv(self.chan.receiver()) -> msg => { @@ -521,18 +520,8 @@ impl Connection { trace!("RPC {:?}", msg); match msg { Message::Request(line) => { - let cmd: Value = from_str(&line).chain_err(|| "invalid JSON format")?; - let reply = match ( - cmd.get("method"), - cmd.get("params").unwrap_or(&empty_params), - cmd.get("id"), - ) { - (Some(Value::String(method)), Value::Array(params), Some(id)) => { - self.handle_command(method, params, id)? - } - _ => bail!("invalid command: {}", cmd), - }; - self.send_values(&[reply])? + let result = self.handle_line(&line); + self.send_values(&[result])? } Message::PeriodicUpdate => { let values = self @@ -554,6 +543,48 @@ impl Connection { } } + #[inline] + fn handle_line(&mut self, line: &String) -> Value { + if let Ok(json_value) = from_str(line) { + match json_value { + Value::Array(mut arr) => { + for cmd in &mut arr { + // Replace each cmd with its response in-memory. + *cmd = self.handle_value(cmd); + } + Value::Array(arr) + } + cmd => self.handle_value(&cmd), + } + } else { + // serde_json was unable to parse + invalid_json_rpc(line) + } + } + + #[inline] + fn handle_value(&mut self, value: &Value) -> Value { + match ( + value.get("method"), + value.get("params").unwrap_or(&json!([])), + value.get("id"), + ) { + (Some(Value::String(method)), Value::Array(params), Some(id)) => self + .handle_command(method, params, id) + .unwrap_or_else(|err| { + json!({ + "error": { + "code": 1, + "message": format!("{method} RPC error: {err}") + }, + "id": id, + "jsonrpc": "2.0" + }) + }), + _ => invalid_json_rpc(value), + } + } + fn handle_requests( mut reader: BufReader, tx: crossbeam_channel::Sender, @@ -629,6 +660,18 @@ impl Connection { } } +#[inline] +fn invalid_json_rpc(input: impl core::fmt::Display) -> Value { + json!({ + "error": { + "code": -32600, + "message": format!("invalid request: {input}") + }, + "id": null, + "jsonrpc": "2.0" + }) +} + fn get_history( query: &Query, scripthash: &[u8], diff --git a/src/elements/asset.rs b/src/elements/asset.rs index b6cd704f..1a3bd24d 100644 --- a/src/elements/asset.rs +++ b/src/elements/asset.rs @@ -71,9 +71,9 @@ pub struct IssuedAsset { #[derive(Serialize, Deserialize, Debug)] pub struct AssetRow { pub issuance_txid: FullHash, - pub issuance_vin: u16, + pub issuance_vin: u32, pub prev_txid: FullHash, - pub prev_vout: u16, + pub prev_vout: u32, pub issuance: Bytes, // bincode does not like dealing with AssetIssuance, deserialization fails with "invalid type: sequence, expected a struct" pub reissuance_token: FullHash, } @@ -105,7 +105,7 @@ impl IssuedAsset { }, issuance_prevout: OutPoint { txid: deserialize(&asset.prev_txid).unwrap(), - vout: asset.prev_vout as u32, + vout: asset.prev_vout, }, contract_hash, reissuance_token, @@ -155,7 +155,7 @@ impl LiquidAsset { #[cfg_attr(test, derive(PartialEq, Eq))] pub struct IssuingInfo { pub txid: FullHash, - pub vin: u16, + pub vin: u32, pub is_reissuance: bool, // None for blinded issuances pub issued_amount: Option, @@ -166,7 +166,7 @@ pub struct IssuingInfo { #[cfg_attr(test, derive(PartialEq, Eq))] pub struct BurningInfo { pub txid: FullHash, - pub vout: u16, + pub vout: u32, pub value: u64, } @@ -174,17 +174,16 @@ pub struct BurningInfo { pub fn index_confirmed_tx_assets( tx: &Transaction, confirmed_height: u32, + tx_position: u16, network: Network, parent_network: BNetwork, rows: &mut Vec, ) { let (history, issuances) = index_tx_assets(tx, network, parent_network); - rows.extend( - history.into_iter().map(|(asset_id, info)| { - asset_history_row(&asset_id, confirmed_height, info).into_row() - }), - ); + rows.extend(history.into_iter().map(|(asset_id, info)| { + asset_history_row(&asset_id, confirmed_height, tx_position, info).into_row() + })); // the initial issuance is kept twice: once in the history index under I, // and once separately under i for asset lookup with some more associated metadata. @@ -205,10 +204,7 @@ pub fn index_mempool_tx_assets( ) { let (history, issuances) = index_tx_assets(tx, network, parent_network); for (asset_id, info) in history { - asset_history - .entry(asset_id) - .or_insert_with(Vec::new) - .push(info); + asset_history.entry(asset_id).or_default().push(info); } for (asset_id, issuance) in issuances { asset_issuance.insert(asset_id, issuance); @@ -251,7 +247,7 @@ fn index_tx_assets( pegout.asset.explicit().unwrap(), TxHistoryInfo::Pegout(PegoutInfo { txid, - vout: txo_index as u16, + vout: txo_index as u32, value: pegout.value, }), )); @@ -262,7 +258,7 @@ fn index_tx_assets( asset_id, TxHistoryInfo::Burning(BurningInfo { txid, - vout: txo_index as u16, + vout: txo_index as u32, value, }), )); @@ -277,7 +273,7 @@ fn index_tx_assets( pegin.asset.explicit().unwrap(), TxHistoryInfo::Pegin(PeginInfo { txid, - vin: txi_index as u16, + vin: txi_index as u32, value: pegin.value, }), )); @@ -302,7 +298,7 @@ fn index_tx_assets( asset_id, TxHistoryInfo::Issuing(IssuingInfo { txid, - vin: txi_index as u16, + vin: txi_index as u32, is_reissuance, issued_amount, token_amount, @@ -319,9 +315,9 @@ fn index_tx_assets( asset_id, AssetRow { issuance_txid: txid, - issuance_vin: txi_index as u16, + issuance_vin: txi_index as u32, prev_txid: full_hash(&txi.previous_output.txid[..]), - prev_vout: txi.previous_output.vout as u16, + prev_vout: txi.previous_output.vout, issuance: serialize(&txi.asset_issuance), reissuance_token: full_hash(&reissuance_token.into_inner()[..]), }, @@ -336,12 +332,14 @@ fn index_tx_assets( fn asset_history_row( asset_id: &AssetId, confirmed_height: u32, + tx_position: u16, txinfo: TxHistoryInfo, ) -> TxHistoryRow { let key = TxHistoryKey { code: b'I', hash: full_hash(&asset_id.into_inner()[..]), confirmed_height, + tx_position, txinfo, }; TxHistoryRow { key } @@ -385,7 +383,7 @@ pub fn lookup_asset( Ok(if let Some(row) = row { let reissuance_token = parse_asset_id(&row.reissuance_token); - let meta = meta.map(Clone::clone).or_else(|| match registry { + let meta = meta.cloned().or_else(|| match registry { Some(AssetRegistryLock::RwLock(rwlock)) => { rwlock.read().unwrap().get(asset_id).cloned() } diff --git a/src/elements/peg.rs b/src/elements/peg.rs index cd339e60..f5eb3d40 100644 --- a/src/elements/peg.rs +++ b/src/elements/peg.rs @@ -19,7 +19,13 @@ pub fn get_pegout_data( let pegged_asset_id = network.pegged_asset()?; txout.pegout_data().filter(|pegout| { pegout.asset == Asset::Explicit(*pegged_asset_id) - && pegout.genesis_hash == bitcoin_genesis_hash(parent_network) + && pegout.genesis_hash + == bitcoin_genesis_hash(match parent_network { + BNetwork::Bitcoin => Network::Liquid, + BNetwork::Testnet => Network::LiquidTestnet, + BNetwork::Signet => return false, + BNetwork::Regtest => Network::LiquidRegtest, + }) }) } @@ -55,7 +61,7 @@ impl PegoutValue { #[cfg_attr(test, derive(PartialEq, Eq))] pub struct PeginInfo { pub txid: FullHash, - pub vin: u16, + pub vin: u32, pub value: u64, } @@ -64,6 +70,6 @@ pub struct PeginInfo { #[cfg_attr(test, derive(PartialEq, Eq))] pub struct PegoutInfo { pub txid: FullHash, - pub vout: u16, + pub vout: u32, pub value: u64, } diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 5229f8e4..94a00137 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -5,7 +5,11 @@ use std::path::Path; use crate::config::Config; use crate::util::{bincode_util, Bytes}; -static DB_VERSION: u32 = 1; +/// Each version will break any running instance with a DB that has a differing version. +/// It will also break if light mode is enabled or disabled. +// 1 = Original DB (since fork from Blockstream) +// 2 = Add tx position to TxHistory rows and place Spending before Funding +static DB_VERSION: u32 = 2; #[derive(Debug, Eq, PartialEq)] pub struct DBRow { diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 64adbd9b..c60e4894 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -9,7 +9,7 @@ use elements::{encode::serialize, AssetId}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::iter::FromIterator; use std::ops::Bound::{Excluded, Unbounded}; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; use crate::chain::{deserialize, Network, OutPoint, Transaction, TxOut, Txid}; @@ -253,7 +253,7 @@ impl Mempool { Some(Utxo { txid: deserialize(&info.txid).expect("invalid txid"), - vout: info.vout as u32, + vout: info.vout, value: info.value, confirmed: None, @@ -386,50 +386,71 @@ impl Mempool { &self.backlog_stats.0 } - pub fn update(&mut self, daemon: &Daemon) -> Result<()> { - let _timer = self.latency.with_label_values(&["update"]).start_timer(); - let new_txids = daemon + pub fn unique_txids(&self) -> HashSet { + return HashSet::from_iter(self.txstore.keys().cloned()); + } + + pub fn update(mempool: &RwLock, daemon: &Daemon) -> Result<()> { + // 1. Start the metrics timer and get the current mempool txids + // [LOCK] Takes read lock for whole scope. + let (_timer, old_txids) = { + let mempool = mempool.read().unwrap(); + ( + mempool.latency.with_label_values(&["update"]).start_timer(), + mempool.unique_txids(), + ) + }; + + // 2. Get all the mempool txids from the RPC. + // [LOCK] No lock taken. Wait for RPC request. Get lists of remove/add txes. + let all_txids = daemon .getmempooltxids() .chain_err(|| "failed to update mempool from daemon")?; - let old_txids = HashSet::from_iter(self.txstore.keys().cloned()); - let to_remove: HashSet<&Txid> = old_txids.difference(&new_txids).collect(); - - // Download and add new transactions from bitcoind's mempool - let txids: Vec<&Txid> = new_txids.difference(&old_txids).collect(); - let to_add = match daemon.gettransactions(&txids) { - Ok(txs) => txs, - Err(err) => { - warn!("failed to get {} transactions: {}", txids.len(), err); // e.g. new block or RBF - return Ok(()); // keep the mempool until next update() + let txids_to_remove: HashSet<&Txid> = old_txids.difference(&all_txids).collect(); + let txids_to_add: Vec<&Txid> = all_txids.difference(&old_txids).collect(); + + // 3. Remove missing transactions. Even if we are unable to download new transactions from + // the daemon, we still want to remove the transactions that are no longer in the mempool. + // [LOCK] Write lock is released at the end of the call to remove(). + mempool.write().unwrap().remove(txids_to_remove); + + // 4. Download the new transactions from the daemon's mempool + // [LOCK] No lock taken, waiting for RPC response. + let txs_to_add = daemon + .gettransactions(&txids_to_add) + .chain_err(|| format!("failed to get {} transactions", txids_to_add.len()))?; + + // 4. Update local mempool to match daemon's state + // [LOCK] Takes Write lock for whole scope. + { + let mut mempool = mempool.write().unwrap(); + // Add new transactions + if txs_to_add.len() > mempool.add(txs_to_add) { + debug!("Mempool update added less transactions than expected"); } - }; - // Add new transactions - if to_add.len() > self.add(to_add) { - debug!("Mempool update added less transactions than expected"); - } - // Remove missing transactions - self.remove(to_remove); - self.count - .with_label_values(&["txs"]) - .set(self.txstore.len() as f64); + mempool + .count + .with_label_values(&["txs"]) + .set(mempool.txstore.len() as f64); + + // Update cached backlog stats (if expired) + if mempool.backlog_stats.1.elapsed() + > Duration::from_secs(mempool.config.mempool_backlog_stats_ttl) + { + let _timer = mempool + .latency + .with_label_values(&["update_backlog_stats"]) + .start_timer(); + mempool.backlog_stats = (BacklogStats::new(&mempool.feeinfo), Instant::now()); + } - // Update cached backlog stats (if expired) - if self.backlog_stats.1.elapsed() - > Duration::from_secs(self.config.mempool_backlog_stats_ttl) - { - let _timer = self - .latency - .with_label_values(&["update_backlog_stats"]) - .start_timer(); - self.backlog_stats = (BacklogStats::new(&self.feeinfo), Instant::now()); + Ok(()) } - - Ok(()) } pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Txid) -> Result<()> { - if self.txstore.get(txid).is_none() { + if !self.txstore.contains_key(txid) { if let Ok(tx) = daemon.getmempooltx(txid) { if self.add(vec![tx]) == 0 { return Err(format!( @@ -461,8 +482,12 @@ impl Mempool { // Phase 1: add to txstore for tx in txs { let txid = tx.txid(); - txids.push(txid); - self.txstore.insert(txid, tx); + // Only push if it doesn't already exist. + // This is important now that update doesn't lock during + // the entire function body. + if self.txstore.insert(txid, tx).is_none() { + txids.push(txid); + } } // Phase 2: index history and spend edges (some txos can be missing) @@ -513,9 +538,9 @@ impl Mempool { compute_script_hash(&prevout.script_pubkey), TxHistoryInfo::Spending(SpendingInfo { txid: txid_bytes, - vin: input_index as u16, + vin: input_index, prev_txid: full_hash(&txi.previous_output.txid[..]), - prev_vout: txi.previous_output.vout as u16, + prev_vout: txi.previous_output.vout, value: prevout.value, }), ) @@ -534,7 +559,7 @@ impl Mempool { compute_script_hash(&txo.script_pubkey), TxHistoryInfo::Funding(FundingInfo { txid: txid_bytes, - vout: index as u16, + vout: index as u32, value: txo.value, }), ) @@ -542,10 +567,7 @@ impl Mempool { // Index funding/spending history entries and spend edges for (scripthash, entry) in funding.chain(spending) { - self.history - .entry(scripthash) - .or_insert_with(Vec::new) - .push(entry); + self.history.entry(scripthash).or_default().push(entry); } for (i, txi) in tx.input.iter().enumerate() { self.edges.insert(txi.previous_output, (txid, i as u32)); diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 3003a256..3e314fd1 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid}; use crate::config::Config; -use crate::daemon::Daemon; +use crate::daemon::{Daemon, MempoolAcceptResult}; use crate::errors::*; use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo}; use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus}; @@ -87,6 +87,14 @@ impl Query { Ok(txid) } + pub fn test_mempool_accept( + &self, + txhex: Vec, + maxfeerate: Option, + ) -> Result> { + self.daemon.test_mempool_accept(txhex, maxfeerate) + } + pub fn utxo(&self, scripthash: &[u8]) -> Result> { let mut utxos = self.chain.utxo( scripthash, diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 97da52df..91131a2a 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -540,21 +540,21 @@ impl ChainQuery { // collate utxo funding/spending events by transaction let rows = iter - .map(|row| (row.get_txid(), row.key.txinfo)) - .skip_while(|(txid, _)| { + .map(|row| (row.get_txid(), row.key.txinfo, row.key.tx_position)) + .skip_while(|(txid, _, _)| { // skip until we reach the last_seen_txid last_seen_txid.map_or(false, |last_seen_txid| last_seen_txid != txid) }) - .skip_while(|(txid, _)| { + .skip_while(|(txid, _, _)| { // skip the last_seen_txid itself last_seen_txid.map_or(false, |last_seen_txid| last_seen_txid == txid) }) - .filter_map(|(txid, info)| { + .filter_map(|(txid, info, tx_position)| { self.tx_confirming_block(&txid) - .map(|b| (txid, info, b.height, b.time)) + .map(|b| (txid, info, b.height, b.time, tx_position)) }); let mut map: HashMap = HashMap::new(); - for (txid, info, height, time) in rows { + for (txid, info, height, time, tx_position) in rows { if !map.contains_key(&txid) && map.len() == limit { break; } @@ -570,6 +570,7 @@ impl ChainQuery { value: info.value.try_into().unwrap_or(0), height, time, + tx_position, }); } #[cfg(not(feature = "liquid"))] @@ -583,6 +584,7 @@ impl ChainQuery { value: 0_i64.saturating_sub(info.value.try_into().unwrap_or(0)), height, time, + tx_position, }); } #[cfg(feature = "liquid")] @@ -592,6 +594,7 @@ impl ChainQuery { value: 0, height, time, + tx_position, }); } #[cfg(feature = "liquid")] @@ -601,6 +604,7 @@ impl ChainQuery { value: 0, height, time, + tx_position, }); } #[cfg(feature = "liquid")] @@ -610,7 +614,11 @@ impl ChainQuery { let mut tx_summaries = map.into_values().collect::>(); tx_summaries.sort_by(|a, b| { if a.height == b.height { - a.value.cmp(&b.value) + if a.tx_position == b.tx_position { + a.value.cmp(&b.value) + } else { + b.tx_position.cmp(&a.tx_position) + } } else { b.height.cmp(&a.height) } @@ -1136,7 +1144,7 @@ impl ChainQuery { let txid: Txid = deserialize(&edge.key.spending_txid).unwrap(); self.tx_confirming_block(&txid).map(|b| SpendingInput { txid, - vin: edge.key.spending_vin as u32, + vin: edge.key.spending_vin, confirmed: Some(b), }) }) @@ -1344,9 +1352,16 @@ fn index_blocks( .par_iter() // serialization is CPU-intensive .map(|b| { let mut rows = vec![]; - for tx in &b.block.txdata { + for (idx, tx) in b.block.txdata.iter().enumerate() { let height = b.entry.height() as u32; - index_transaction(tx, height, previous_txos_map, &mut rows, iconfig); + index_transaction( + tx, + height, + idx as u16, + previous_txos_map, + &mut rows, + iconfig, + ); } rows.push(BlockRow::new_done(full_hash(&b.entry.hash()[..])).into_row()); // mark block as "indexed" rows @@ -1359,13 +1374,14 @@ fn index_blocks( fn index_transaction( tx: &Transaction, confirmed_height: u32, + tx_position: u16, previous_txos_map: &HashMap, rows: &mut Vec, iconfig: &IndexerConfig, ) { // persist history index: - // H{funding-scripthash}{funding-height}F{funding-txid:vout} → "" - // H{funding-scripthash}{spending-height}S{spending-txid:vin}{funding-txid:vout} → "" + // H{funding-scripthash}{spending-height}{spending-block-pos}S{spending-txid:vin}{funding-txid:vout} → "" + // H{funding-scripthash}{funding-height}{funding-block-pos}F{funding-txid:vout} → "" // persist "edges" for fast is-this-TXO-spent check // S{funding-txid:vout}{spending-txid:vin} → "" let txid = full_hash(&tx.txid()[..]); @@ -1374,9 +1390,10 @@ fn index_transaction( let history = TxHistoryRow::new( &txo.script_pubkey, confirmed_height, + tx_position, TxHistoryInfo::Funding(FundingInfo { txid, - vout: txo_index as u16, + vout: txo_index as u32, value: txo.value, }), ); @@ -1400,11 +1417,12 @@ fn index_transaction( let history = TxHistoryRow::new( &prev_txo.script_pubkey, confirmed_height, + tx_position, TxHistoryInfo::Spending(SpendingInfo { txid, - vin: txi_index as u16, + vin: txi_index as u32, prev_txid: full_hash(&txi.previous_output.txid[..]), - prev_vout: txi.previous_output.vout as u16, + prev_vout: txi.previous_output.vout, value: prev_txo.value, }), ); @@ -1412,9 +1430,9 @@ fn index_transaction( let edge = TxEdgeRow::new( full_hash(&txi.previous_output.txid[..]), - txi.previous_output.vout as u16, + txi.previous_output.vout, txid, - txi_index as u16, + txi_index as u32, ); rows.push(edge.into_row()); } @@ -1424,6 +1442,7 @@ fn index_transaction( asset::index_confirmed_tx_assets( tx, confirmed_height, + tx_position, iconfig.network, iconfig.parent_network, rows, @@ -1534,7 +1553,7 @@ impl TxConfRow { struct TxOutKey { code: u8, txid: FullHash, - vout: u16, + vout: u32, } struct TxOutRow { @@ -1548,7 +1567,7 @@ impl TxOutRow { key: TxOutKey { code: b'O', txid: *txid, - vout: vout as u16, + vout: vout as u32, }, value: serialize(txout), } @@ -1557,7 +1576,7 @@ impl TxOutRow { bincode_util::serialize_little(&TxOutKey { code: b'O', txid: full_hash(&outpoint.txid[..]), - vout: outpoint.vout as u16, + vout: outpoint.vout, }) .unwrap() } @@ -1648,7 +1667,7 @@ impl BlockRow { #[cfg_attr(test, derive(PartialEq, Eq))] pub struct FundingInfo { pub txid: FullHash, - pub vout: u16, + pub vout: u32, pub value: Value, } @@ -1656,17 +1675,20 @@ pub struct FundingInfo { #[cfg_attr(test, derive(PartialEq, Eq))] pub struct SpendingInfo { pub txid: FullHash, // spending transaction - pub vin: u16, + pub vin: u32, pub prev_txid: FullHash, // funding transaction - pub prev_vout: u16, + pub prev_vout: u32, pub value: Value, } #[derive(Serialize, Deserialize, Debug)] #[cfg_attr(test, derive(PartialEq, Eq))] pub enum TxHistoryInfo { - Funding(FundingInfo), + // If a spend and a fund for the same scripthash + // occur in the same tx, spends should come first. + // This ordering comes from the enum order. Spending(SpendingInfo), + Funding(FundingInfo), #[cfg(feature = "liquid")] Issuing(asset::IssuingInfo), @@ -1700,6 +1722,7 @@ pub struct TxHistoryKey { pub code: u8, // H for script history or I for asset history (elements only) pub hash: FullHash, // either a scripthash (always on bitcoin) or an asset id (elements only) pub confirmed_height: u32, // MUST be serialized as big-endian (for correct scans). + pub tx_position: u16, // MUST be serialized as big-endian (for correct scans). Position in block. pub txinfo: TxHistoryInfo, } @@ -1708,11 +1731,17 @@ pub struct TxHistoryRow { } impl TxHistoryRow { - fn new(script: &Script, confirmed_height: u32, txinfo: TxHistoryInfo) -> Self { + fn new( + script: &Script, + confirmed_height: u32, + tx_position: u16, + txinfo: TxHistoryInfo, + ) -> Self { let key = TxHistoryKey { code: b'H', hash: compute_script_hash(script), confirmed_height, + tx_position, txinfo, }; TxHistoryRow { key } @@ -1723,7 +1752,7 @@ impl TxHistoryRow { } fn prefix_end(code: u8, hash: &[u8]) -> Bytes { - bincode_util::serialize_big(&(code, full_hash(hash), std::u32::MAX)).unwrap() + bincode_util::serialize_big(&(code, full_hash(hash), u32::MAX)).unwrap() } fn prefix_height(code: u8, hash: &[u8], height: u32) -> Bytes { @@ -1758,11 +1787,11 @@ impl TxHistoryInfo { match self { TxHistoryInfo::Funding(ref info) => OutPoint { txid: deserialize(&info.txid).unwrap(), - vout: info.vout as u32, + vout: info.vout, }, TxHistoryInfo::Spending(ref info) => OutPoint { txid: deserialize(&info.prev_txid).unwrap(), - vout: info.prev_vout as u32, + vout: info.prev_vout, }, #[cfg(feature = "liquid")] TxHistoryInfo::Issuing(_) @@ -1779,15 +1808,16 @@ pub struct TxHistorySummary { height: usize, value: i64, time: u32, + tx_position: u16, } #[derive(Serialize, Deserialize)] struct TxEdgeKey { code: u8, funding_txid: FullHash, - funding_vout: u16, + funding_vout: u32, spending_txid: FullHash, - spending_vin: u16, + spending_vin: u32, } struct TxEdgeRow { @@ -1797,9 +1827,9 @@ struct TxEdgeRow { impl TxEdgeRow { fn new( funding_txid: FullHash, - funding_vout: u16, + funding_vout: u32, spending_txid: FullHash, - spending_vin: u16, + spending_vin: u32, ) -> Self { let key = TxEdgeKey { code: b'S', @@ -1813,7 +1843,7 @@ impl TxEdgeRow { fn filter(outpoint: &OutPoint) -> Bytes { // TODO build key without using bincode? [ b"S", &outpoint.txid[..], outpoint.vout?? ].concat() - bincode_util::serialize_little(&(b'S', full_hash(&outpoint.txid[..]), outpoint.vout as u16)) + bincode_util::serialize_little(&(b'S', full_hash(&outpoint.txid[..]), outpoint.vout)) .unwrap() } @@ -1943,14 +1973,16 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // confirmed_height 0, 0, 0, 2, + // tx_position + 0, 3, // TxHistoryInfo variant (Funding) - 0, 0, 0, 0, + 0, 0, 0, 1, // FundingInfo // txid 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // vout - 0, 3, + 0, 0, 0, 3, // Value variant (Explicit) 0, 0, 0, 0, 0, 0, 0, 2, // number of tuple elements @@ -1963,10 +1995,11 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 2, - 0, 0, 0, 0, + 0, 3, + 0, 0, 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 0, 3, + 0, 0, 0, 3, // Value variant (Null) 0, 0, 0, 0, 0, 0, 0, 1, // number of tuple elements @@ -1977,13 +2010,14 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 2, - 0, 0, 0, 1, + 0, 3, + 0, 0, 0, 0, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 0, 12, + 0, 0, 0, 12, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, - 0, 9, + 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 2, 1, 14, 0, 0, 0, 0, 0, 0, 0, @@ -1993,13 +2027,14 @@ mod tests { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 2, - 0, 0, 0, 1, + 0, 3, + 0, 0, 0, 0, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 0, 12, + 0, 0, 0, 12, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, 98, 101, 101, 102, - 0, 9, + 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, ], @@ -2010,6 +2045,7 @@ mod tests { code: b'H', hash: [1; 32], confirmed_height: 2, + tx_position: 3, txinfo: super::TxHistoryInfo::Funding(super::FundingInfo { txid: [2; 32], vout: 3, @@ -2022,6 +2058,7 @@ mod tests { code: b'H', hash: [1; 32], confirmed_height: 2, + tx_position: 3, txinfo: super::TxHistoryInfo::Funding(super::FundingInfo { txid: [2; 32], vout: 3, @@ -2034,6 +2071,7 @@ mod tests { code: b'H', hash: [1; 32], confirmed_height: 2, + tx_position: 3, txinfo: super::TxHistoryInfo::Spending(super::SpendingInfo { txid: [18; 32], vin: 12, @@ -2048,6 +2086,7 @@ mod tests { code: b'H', hash: [1; 32], confirmed_height: 2, + tx_position: 3, txinfo: super::TxHistoryInfo::Spending(super::SpendingInfo { txid: [18; 32], vin: 12, diff --git a/src/rest.rs b/src/rest.rs index 49900b36..f6e790c4 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -355,6 +355,8 @@ impl TxOutValue { "v0_p2wsh" } else if is_v1_p2tr(script) { "v1_p2tr" + } else if is_anchor(script) { + "anchor" } else if script.is_provably_unspendable() { "provably_unspendable" } else if is_bare_multisig(script) { @@ -405,6 +407,15 @@ fn is_bare_multisig(script: &Script) -> bool { && script[0] <= script[len - 2] } +fn is_anchor(script: &Script) -> bool { + let len = script.len(); + len == 4 + && script[0] == opcodes::all::OP_PUSHNUM_1.into_u8() + && script[1] == opcodes::all::OP_PUSHBYTES_2.into_u8() + && script[2] == 0x4e + && script[3] == 0x73 +} + #[derive(Serialize)] struct UtxoValue { txid: Txid, @@ -933,7 +944,7 @@ fn handle_request( _ => "", }; let script_hashes: Vec<[u8; 32]> = query_params - .get(&script_types.to_string()) + .get(*script_types) .ok_or(HttpError::from(format!("No {} specified", script_types)))? .as_str() .split(',') @@ -1085,7 +1096,7 @@ fn handle_request( _ => "", }; let script_hashes: Vec<[u8; 32]> = query_params - .get(&script_types.to_string()) + .get(*script_types) .ok_or(HttpError::from(format!("No {} specified", script_types)))? .as_str() .split(',') @@ -1322,6 +1333,48 @@ fn handle_request( .map_err(|err| HttpError::from(err.description().to_string()))?; http_message(StatusCode::OK, txid.to_hex(), 0) } + (&Method::POST, Some(&"txs"), Some(&"test"), None, None, None) => { + let txhexes: Vec = + serde_json::from_str(String::from_utf8(body.to_vec())?.as_str())?; + + if txhexes.len() > 25 { + Result::Err(HttpError::from( + "Exceeded maximum of 25 transactions".to_string(), + ))? + } + + let maxfeerate = query_params + .get("maxfeerate") + .map(|s| { + s.parse::() + .map_err(|_| HttpError::from("Invalid maxfeerate".to_string())) + }) + .transpose()?; + + // pre-checks + txhexes.iter().enumerate().try_for_each(|(index, txhex)| { + // each transaction must be of reasonable size (more than 60 bytes, within 400kWU standardness limit) + if !(120..800_000).contains(&txhex.len()) { + Result::Err(HttpError::from(format!( + "Invalid transaction size for item {}", + index + ))) + } else { + // must be a valid hex string + Vec::::from_hex(txhex) + .map_err(|_| { + HttpError::from(format!("Invalid transaction hex for item {}", index)) + }) + .map(|_| ()) + } + })?; + + let result = query + .test_mempool_accept(txhexes, maxfeerate) + .map_err(|err| HttpError::from(err.description().to_string()))?; + + json_response(result, TTL_SHORT) + } (&Method::GET, Some(&"txs"), Some(&"outspends"), None, None, None) => { let txid_strings: Vec<&str> = query_params .get("txids") @@ -1746,7 +1799,10 @@ fn address_to_scripthash(addr: &str, network: Network) -> Result self.len() - 1 { 0 } else { diff --git a/src/util/transaction.rs b/src/util/transaction.rs index c9ff29ae..d1a8cc2c 100644 --- a/src/util/transaction.rs +++ b/src/util/transaction.rs @@ -48,7 +48,7 @@ impl From> for TransactionStatus { #[derive(Serialize, Deserialize)] pub struct TxInput { pub txid: Txid, - pub vin: u16, + pub vin: u32, } pub fn is_coinbase(txin: &TxIn) -> bool { @@ -340,18 +340,12 @@ pub(super) mod sigops { let last_witness = witness.last(); match (witness_version, witness_program.len()) { (0, 20) => 1, - (0, 32) => { - if let Some(n) = last_witness - .map(|sl| sl.iter().map(|v| Ok(*v))) - .map(script::Script::from_byte_iter) - // I only return Ok 2 lines up, so there is no way to error - .map(|s| count_sigops(&s.unwrap(), true)) - { - n - } else { - 0 - } - } + (0, 32) => last_witness + .map(|sl| sl.iter().map(|v| Ok(*v))) + .map(script::Script::from_byte_iter) + // I only return Ok 2 lines up, so there is no way to error + .map(|s| count_sigops(&s.unwrap(), true)) + .unwrap_or_default(), _ => 0, } } diff --git a/start b/start index 16139f7b..7e5cd80f 100755 --- a/start +++ b/start @@ -43,6 +43,11 @@ case "${1}" in NETWORK=testnet THREADS=$((NPROC / 6)) ;; + testnet4) + NETWORK=testnet4 + MAGIC=283f161c + THREADS=$((NPROC / 6)) + ;; signet) NETWORK=signet THREADS=$((NPROC / 6)) @@ -60,7 +65,7 @@ case "${1}" in THREADS=$((NPROC / 6)) ;; *) - echo "Usage: $0 (mainnet|testnet|signet|liquid|liquidtestnet)" + echo "Usage: $0 (mainnet|testnet|testnet4|signet|liquid|liquidtestnet)" exit 1 ;; esac @@ -93,6 +98,22 @@ do ELECTRUM_TXS_LIMIT=9000 MAIN_LOOP_DELAY=14000 fi + if [ "${NODENAME}" = "node213" ];then + UTXOS_LIMIT=9000 + ELECTRUM_TXS_LIMIT=9000 + fi + if [ "${NODENAME}" = "node213" ];then + UTXOS_LIMIT=9000 + ELECTRUM_TXS_LIMIT=9000 + fi + if [ "${NETWORK}" = "testnet4" ];then + UTXOS_LIMIT=9000 + ELECTRUM_TXS_LIMIT=9000 + fi + if [ "${LOCATION}" = "fmt" ];then + UTXOS_LIMIT=9000 + ELECTRUM_TXS_LIMIT=9000 + fi # Run the popular address txt file generator before each run POPULAR_SCRIPTS_FOLDER="${HOME}/popular-scripts/${NETWORK}" @@ -136,6 +157,7 @@ do --precache-threads "${THREADS}" \ --cookie "${RPC_USER}:${RPC_PASS}" \ --cors '*' \ + --magic "${MAGIC}" \ --address-search \ --utxos-limit "${UTXOS_LIMIT}" \ --electrum-txs-limit "${ELECTRUM_TXS_LIMIT}" \