From 07b2651e94cd1efeec7c8a29cea175dae60aa145 Mon Sep 17 00:00:00 2001 From: Michael Mallan Date: Fri, 15 Nov 2024 12:20:13 +0000 Subject: [PATCH] sqlite: store if coin is from self This adds a new `is_from_self` column to the coins table that indicates if the coin is an output of a transaction for which all inputs are ours. In the case of unconfirmed coins, being from self requires that all unconfirmed ancestors, if any, are also from self. The DB migration sets the value of this column for all existing coins and the poller keeps the column up to date. --- lianad/src/bitcoin/poller/looper.rs | 3 + lianad/src/database/mod.rs | 9 + lianad/src/database/sqlite/mod.rs | 380 ++++++++++++++++++++++++- lianad/src/database/sqlite/schema.rs | 12 +- lianad/src/database/sqlite/utils.rs | 397 +++++++++++++++++++++++++++ lianad/src/testutils.rs | 4 + 6 files changed, 790 insertions(+), 15 deletions(-) diff --git a/lianad/src/bitcoin/poller/looper.rs b/lianad/src/bitcoin/poller/looper.rs index baa95c2cf..181d3c9d7 100644 --- a/lianad/src/bitcoin/poller/looper.rs +++ b/lianad/src/bitcoin/poller/looper.rs @@ -315,6 +315,9 @@ fn updates( db_conn.unspend_coins(&updated_coins.expired_spending); db_conn.spend_coins(&updated_coins.spending); db_conn.confirm_spend(&updated_coins.spent); + // Update info about which coins are from self only after + // coins have been inserted & updated above. + db_conn.update_coins_from_self(current_tip.height); if latest_tip != current_tip { db_conn.update_tip(&latest_tip); log::debug!("New tip: '{}'", latest_tip); diff --git a/lianad/src/database/mod.rs b/lianad/src/database/mod.rs index 6ffce1e3a..2057b4483 100644 --- a/lianad/src/database/mod.rs +++ b/lianad/src/database/mod.rs @@ -181,6 +181,10 @@ pub trait DatabaseConnection { /// Store transactions in database, ignoring any that already exist. fn new_txs(&mut self, txs: &[bitcoin::Transaction]); + /// For all unconfirmed coins and those confirmed after `current_tip_height`, + /// update whether the coin is from self or not. + fn update_coins_from_self(&mut self, current_tip_height: i32); + /// Retrieve a list of transactions and their corresponding block heights and times. fn list_wallet_transactions( &mut self, @@ -379,6 +383,11 @@ impl DatabaseConnection for SqliteConn { self.new_txs(txs) } + fn update_coins_from_self(&mut self, current_tip_height: i32) { + self.update_coins_from_self(current_tip_height) + .expect("must not fail") + } + fn list_wallet_transactions( &mut self, txids: &[bitcoin::Txid], diff --git a/lianad/src/database/sqlite/mod.rs b/lianad/src/database/sqlite/mod.rs index ae227a075..79c8431d4 100644 --- a/lianad/src/database/sqlite/mod.rs +++ b/lianad/src/database/sqlite/mod.rs @@ -43,7 +43,7 @@ use miniscript::bitcoin::{ secp256k1, }; -const DB_VERSION: i64 = 6; +const DB_VERSION: i64 = 7; /// Last database version for which Bitcoin transactions were not stored in database. In practice /// this meant we relied on the bitcoind watchonly wallet to store them for us. @@ -461,8 +461,8 @@ impl SqliteConn { for coin in coins { let deriv_index: u32 = coin.derivation_index.into(); db_tx.execute( - "INSERT INTO coins (wallet_id, txid, vout, amount_sat, derivation_index, is_change, is_immature) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + "INSERT INTO coins (wallet_id, txid, vout, amount_sat, derivation_index, is_change, is_immature, is_from_self) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", rusqlite::params![ WALLET_ID, coin.outpoint.txid[..].to_vec(), @@ -471,6 +471,7 @@ impl SqliteConn { deriv_index, coin.is_change, coin.is_immature, + false, // new coins are inserted as not from self and then updated later ], )?; } @@ -733,9 +734,9 @@ impl SqliteConn { let txid = &tx.txid()[..].to_vec(); let tx_ser = bitcoin::consensus::serialize(tx); db_tx.execute( - "INSERT INTO transactions (txid, tx) VALUES (?1, ?2) \ + "INSERT INTO transactions (txid, tx, num_inputs) VALUES (?1, ?2, ?3) \ ON CONFLICT DO NOTHING", - rusqlite::params![txid, tx_ser,], + rusqlite::params![txid, tx_ser, tx.input.len()], )?; } Ok(()) @@ -743,6 +744,17 @@ impl SqliteConn { .expect("Database must be available") } + // Update `is_from_self` in coins table for all unconfirmed coins + // and those confirmed after `current_tip_height`. + pub fn update_coins_from_self( + &mut self, + current_tip_height: i32, + ) -> Result<(), rusqlite::Error> { + db_exec(&mut self.conn, |db_tx| { + utils::update_coins_from_self(db_tx, current_tip_height) + }) + } + pub fn list_wallet_transactions( &mut self, txids: &[bitcoin::Txid], @@ -807,6 +819,10 @@ impl SqliteConn { /// - Spending transactions confirmation /// - Tip /// + /// The `is_from_self` value for all unconfirmed coins following the rollback is + /// set to false. This is because this value depends on the confirmation status + /// of ancestor coins and so will need to be re-evaluated. + /// /// This will have to be updated if we are to add new fields based on block data /// in the database eventually. pub fn rollback_tip(&mut self, new_tip: &BlockChainTip) { @@ -819,6 +835,12 @@ impl SqliteConn { "UPDATE coins SET spend_block_height = NULL, spend_block_time = NULL WHERE spend_block_height > ?1", rusqlite::params![new_tip.height], )?; + // This statement must be run after updating `blockheight` above so that it includes coins + // that become unconfirmed following the rollback. + db_tx.execute( + "UPDATE coins SET is_from_self = 0 WHERE blockheight IS NULL", + rusqlite::params![], + )?; db_tx.execute( "UPDATE tip SET blockheight = (?1), blockhash = (?2)", rusqlite::params![new_tip.height, new_tip.hash[..].to_vec()], @@ -840,7 +862,7 @@ mod tests { str::FromStr, }; - use bitcoin::bip32; + use bitcoin::{bip32, BlockHash, TxIn}; // The database schema used by the first versions of Liana (database version 0). Used to test // migrations starting from the first version. @@ -2457,7 +2479,332 @@ CREATE TABLE labels ( } #[test] - fn v0_to_v6_migration() { + fn sqlite_update_coins_from_self() { + let (tmp_dir, _, _, db) = dummy_db(); + + // Helper to create a dummy transaction. + // Varying `lock_time_height` allows to obtain a unique txid for the given `num_inputs`. + fn dummy_tx(num_inputs: u32, lock_time_height: u32) -> bitcoin::Transaction { + bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(lock_time_height).unwrap(), + input: (0..num_inputs).map(|_| TxIn::default()).collect(), + output: Vec::new(), + } + } + + { + let mut conn = db.connection().unwrap(); + + // Deposit two coins from two different external transactions. + let tx_a = dummy_tx(1, 0); + let tx_b = dummy_tx(1, 1); + let coin_tx_a: Coin = Coin { + outpoint: bitcoin::OutPoint::new(tx_a.txid(), 0), + is_immature: false, + amount: bitcoin::Amount::from_sat(1_000_000), + derivation_index: bip32::ChildNumber::from_normal_idx(0).unwrap(), + is_change: false, + block_info: None, + spend_txid: None, + spend_block: None, + }; + let coin_tx_b: Coin = Coin { + outpoint: bitcoin::OutPoint::new(tx_b.txid(), 0), + is_immature: false, + amount: bitcoin::Amount::from_sat(1_000_000), + derivation_index: bip32::ChildNumber::from_normal_idx(1).unwrap(), + is_change: false, + block_info: None, + spend_txid: None, + spend_block: None, + }; + conn.new_txs(&[tx_a, tx_b]); + conn.new_unspent_coins(&[coin_tx_a, coin_tx_b]); + + // The coins are not from self. + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + // Update from self info. + conn.update_coins_from_self(0).unwrap(); + // As expected, the coins are still not marked as from self. + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + + // Spend `coin_tx_a` in `tx_c` with change `coin_tx_c`. + let tx_c = dummy_tx(1, 2); + let coin_tx_c: Coin = Coin { + outpoint: bitcoin::OutPoint::new(tx_c.txid(), 0), + is_immature: false, + amount: bitcoin::Amount::from_sat(1_000_000), + derivation_index: bip32::ChildNumber::from_normal_idx(2).unwrap(), + is_change: true, + block_info: None, + spend_txid: None, + spend_block: None, + }; + conn.new_txs(&[tx_c.clone()]); + conn.spend_coins(&[(coin_tx_a.outpoint, tx_c.txid())]); + conn.new_unspent_coins(&[coin_tx_c]); + + // Although `coin_tx_c` has only one parent, `coin_tx_a` is + // unconfirmed and not from self. So all our coins are still + // not marked as from self. + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + conn.update_coins_from_self(0).unwrap(); + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + + // Now refresh `coin_tx_c` in `tx_d`, creating `coin_tx_d`. + let tx_d = dummy_tx(1, 3); + let coin_tx_d: Coin = Coin { + outpoint: bitcoin::OutPoint::new(tx_d.txid(), 0), + is_immature: false, + amount: bitcoin::Amount::from_sat(1_000_000), + derivation_index: bip32::ChildNumber::from_normal_idx(3).unwrap(), + is_change: true, + block_info: None, + spend_txid: None, + spend_block: None, + }; + conn.new_txs(&[tx_d.clone()]); + conn.spend_coins(&[(coin_tx_c.outpoint, tx_d.txid())]); + conn.new_unspent_coins(&[coin_tx_d]); + + // All coins are unconfirmed and none are from self. + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + conn.update_coins_from_self(0).unwrap(); + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + + // Spend the deposited coin `coin_tx_b` and the refreshed coin `coin_tx_d` + // together in `tx_e`, creating `coin_tx_e`. + let tx_e = dummy_tx(2, 4); // 2 inputs + let coin_tx_e: Coin = Coin { + outpoint: bitcoin::OutPoint::new(tx_e.txid(), 0), + is_immature: false, + amount: bitcoin::Amount::from_sat(1_000_000), + derivation_index: bip32::ChildNumber::from_normal_idx(4).unwrap(), + is_change: false, + block_info: None, + spend_txid: None, + spend_block: None, + }; + conn.new_txs(&[tx_e.clone()]); + conn.spend_coins(&[ + (coin_tx_b.outpoint, tx_e.txid()), + (coin_tx_d.outpoint, tx_e.txid()), + ]); + conn.new_unspent_coins(&[coin_tx_e]); + + // Still there are no confirmed coins, so everything remains as not from self. + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + conn.update_coins_from_self(0).unwrap(); + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + + // Finally, refresh `coin_tx_e` in transaction `tx_f`, creating `coin_tx_f`. + let tx_f = dummy_tx(1, 5); + let coin_tx_f: Coin = Coin { + outpoint: bitcoin::OutPoint::new(tx_f.txid(), 0), + is_immature: false, + amount: bitcoin::Amount::from_sat(1_000_000), + derivation_index: bip32::ChildNumber::from_normal_idx(5).unwrap(), + is_change: true, + block_info: None, + spend_txid: None, + spend_block: None, + }; + conn.new_txs(&[tx_f.clone()]); + conn.spend_coins(&[(coin_tx_e.outpoint, tx_f.txid())]); + conn.new_unspent_coins(&[coin_tx_f]); + + // Still no coins are from self. + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + conn.update_coins_from_self(0).unwrap(); + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + + // Now confirm `tx_a` and `tx_c` in successive blocks. + conn.confirm_coins(&[ + (coin_tx_a.outpoint, 100, 1_000), + (coin_tx_c.outpoint, 101, 1_001), + ]); + conn.confirm_spend(&[(coin_tx_a.outpoint, tx_c.txid(), 101, 1_001)]); + // Coins are still not marked as from self. + assert!(conn.coins(&[], &[]).iter().all(|c| !c.is_from_self)); + // Now update from self for coins confirmed after 101, which excludes the two coins above. + // Only `coin_tx_d` is from self, because it's unconfirmed and its parent is a confirmed coin. + // `coin_tx_e` still depends on `coin_tx_b` which is an unconfirmed deposit. + conn.update_coins_from_self(101).unwrap(); + assert!(conn + .coins(&[], &[coin_tx_d.outpoint]) + .iter() + .all(|c| c.is_from_self)); + assert!(conn + .coins( + &[], + &[ + coin_tx_a.outpoint, + coin_tx_b.outpoint, + coin_tx_c.outpoint, + coin_tx_e.outpoint, + coin_tx_f.outpoint + ] + ) + .iter() + .all(|c| !c.is_from_self)); + + // Now run the update for coins confirmed after 100. + conn.update_coins_from_self(100).unwrap(); + // `coin_tx_c` is now marked as from self as it has a single parent + // that is confirmed (even though its parent is an external deposit). + assert!(conn + .coins(&[], &[coin_tx_c.outpoint, coin_tx_d.outpoint]) + .iter() + .all(|c| c.is_from_self)); + assert!(conn + .coins( + &[], + &[ + coin_tx_a.outpoint, + coin_tx_b.outpoint, + coin_tx_e.outpoint, + coin_tx_f.outpoint + ] + ) + .iter() + .all(|c| !c.is_from_self)); + + // Even if we run the update for coins confirmed after height 99, + // `coin_tx_a` will not be marked as from self as it's an external deposit. + conn.update_coins_from_self(99).unwrap(); + assert!(conn + .coins(&[], &[coin_tx_c.outpoint, coin_tx_d.outpoint]) + .iter() + .all(|c| c.is_from_self)); + assert!(conn + .coins( + &[], + &[ + coin_tx_a.outpoint, + coin_tx_b.outpoint, + coin_tx_e.outpoint, + coin_tx_f.outpoint + ] + ) + .iter() + .all(|c| !c.is_from_self)); + + // Now confirm the other external deposit coin. + conn.confirm_coins(&[(coin_tx_b.outpoint, 102, 1_002)]); + // If we run the update, it doesn't matter if we use a later height + // as there are only unconfirmed coins that need to be updated. + conn.update_coins_from_self(110).unwrap(); + // `coin_tx_e` and `coin_tx_f` are also now from self. + assert!(conn + .coins( + &[], + &[ + coin_tx_c.outpoint, + coin_tx_d.outpoint, + coin_tx_e.outpoint, + coin_tx_f.outpoint + ] + ) + .iter() + .all(|c| c.is_from_self)); + assert!(conn + .coins(&[], &[coin_tx_a.outpoint, coin_tx_b.outpoint,]) + .iter() + .all(|c| !c.is_from_self)); + + // Even if we now run the update with an earlier height, + // `coin_tx_b` will not be marked as from self. + conn.update_coins_from_self(101).unwrap(); + // No changes from above. + assert!(conn + .coins( + &[], + &[ + coin_tx_c.outpoint, + coin_tx_d.outpoint, + coin_tx_e.outpoint, + coin_tx_f.outpoint + ] + ) + .iter() + .all(|c| c.is_from_self)); + assert!(conn + .coins(&[], &[coin_tx_a.outpoint, coin_tx_b.outpoint,]) + .iter() + .all(|c| !c.is_from_self)); + + // Now we will roll the tip back earlier than some of our confirmed coins. + let new_tip = { + // It doesn't matter what this hash value is as we only care about the height. + let hash = BlockHash::from_str( + "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f", + ) + .unwrap(); + &BlockChainTip { height: 101, hash } + }; + conn.rollback_tip(new_tip); + + // Only `coin_tx_a` and `coin_tx_c` are still confirmed. + assert_eq!( + conn.coins(&[], &[]) + .iter() + .filter_map(|c| if c.block_info.is_some() { + Some(c.outpoint) + } else { + None + }) + .collect::>(), + vec![coin_tx_a.outpoint, coin_tx_c.outpoint] + ); + // Rolling back sets all unconfirmed coins as not from self so only + // `coin_tx_c` is still marked as from self. + assert!(conn + .coins(&[], &[coin_tx_c.outpoint]) + .iter() + .all(|c| c.is_from_self)); + assert!(conn + .coins( + &[], + &[ + coin_tx_a.outpoint, + coin_tx_b.outpoint, + coin_tx_d.outpoint, + coin_tx_e.outpoint, + coin_tx_f.outpoint + ] + ) + .iter() + .all(|c| !c.is_from_self)); + + // Now run the update from the current tip height of 101. + conn.update_coins_from_self(101).unwrap(); + // `coin_tx_d` is now marked as from self as its parent `coin_tx_c` + // is confirmed. Coins `coin_tx_e` and `coin_tx_f` depend on the + // unconfirmed `tx_coin_b` and so remain as not from self. + assert!(conn + .coins(&[], &[coin_tx_c.outpoint, coin_tx_d.outpoint]) + .iter() + .all(|c| c.is_from_self)); + assert!(conn + .coins( + &[], + &[ + coin_tx_a.outpoint, + coin_tx_b.outpoint, + coin_tx_e.outpoint, + coin_tx_f.outpoint, + ] + ) + .iter() + .all(|c| !c.is_from_self)); + } + + fs::remove_dir_all(tmp_dir).unwrap(); + } + + #[test] + fn v0_to_v7_migration() { let secp = secp256k1::Secp256k1::verification_only(); // Create a database with version 0, using the old schema. @@ -2563,7 +2910,7 @@ CREATE TABLE labels ( { let mut conn = db.connection().unwrap(); let version = conn.db_version(); - assert_eq!(version, 6); + assert_eq!(version, 7); } // We should now be able to insert another PSBT, to query both, and the first PSBT must // have no associated timestamp. @@ -2637,7 +2984,7 @@ CREATE TABLE labels ( } #[test] - fn v3_to_v6_migration() { + fn v3_to_v7_migration() { let secp = secp256k1::Secp256k1::verification_only(); // Create a database with version 3, using the old schema. @@ -2660,7 +3007,7 @@ CREATE TABLE labels ( .map(|i| bitcoin::Transaction { version: bitcoin::transaction::Version::TWO, lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), - input: Vec::new(), + input: vec![TxIn::default()], // a single input output: Vec::new(), }) .collect(); @@ -2787,10 +3134,10 @@ CREATE TABLE labels ( // Migrate the DB. maybe_apply_migration(&db_path, &bitcoin_txs).unwrap(); - assert_eq!(conn.db_version(), 6); + assert_eq!(conn.db_version(), 7); // Migrating twice will be a no-op. No need to pass `bitcoin_txs` second time. maybe_apply_migration(&db_path, &[]).unwrap(); - assert!(conn.db_version() == 6); + assert!(conn.db_version() == 7); // Compare the `DbCoin`s with the expected values. let coins_post = conn.coins(&[], &[]); @@ -2809,6 +3156,11 @@ CREATE TABLE labels ( assert_eq!(c_post.is_change, c_pre.is_change); assert_eq!(c_post.spend_txid, c_pre.spend_txid); assert_eq!(c_post.spend_block, c_pre.spend_block); + // only coins D and E are from self. + assert_eq!( + c_post.is_from_self, + [coin_d_outpoint, coin_e_outpoint].contains(&c_pre.outpoint) + ); } } @@ -2850,7 +3202,7 @@ CREATE TABLE labels ( }, if i % 2 == 0 { Some(DbBlockInfo { - height: (i % 5) as i32 * 2_000, + height: 1 + (i % 5) as i32 * 2_000, time: 1722488619 + (i % 5) * 84_999, }) } else { @@ -2878,7 +3230,7 @@ CREATE TABLE labels ( is_change: (i % 4) == 0, block_info: if i & 2 == 0 { Some(DbBlockInfo { - height: (i % 100) as i32 * 1_000, + height: 1 + (i % 100) as i32 * 1_000, time: 1722408619 + (i % 100) as u32 * 42_000, }) } else { diff --git a/lianad/src/database/sqlite/schema.rs b/lianad/src/database/sqlite/schema.rs index 52a83a0be..ee5485789 100644 --- a/lianad/src/database/sqlite/schema.rs +++ b/lianad/src/database/sqlite/schema.rs @@ -57,6 +57,7 @@ CREATE TABLE coins ( spend_block_height INTEGER, spend_block_time INTEGER, is_immature BOOLEAN NOT NULL CHECK (is_immature IN (0,1)), + is_from_self BOOLEAN NOT NULL CHECK (is_from_self IN (0,1)), UNIQUE (txid, vout), FOREIGN KEY (wallet_id) REFERENCES wallets (id) ON UPDATE RESTRICT @@ -82,7 +83,8 @@ CREATE TABLE addresses ( CREATE TABLE transactions ( id INTEGER PRIMARY KEY NOT NULL, txid BLOB UNIQUE NOT NULL, - tx BLOB UNIQUE NOT NULL + tx BLOB UNIQUE NOT NULL, + num_inputs INTEGER NOT NULL ); /* Transactions we created that spend some of our coins. */ @@ -195,6 +197,12 @@ pub struct DbCoin { pub is_change: bool, pub spend_txid: Option, pub spend_block: Option, + /// A coin is from self if it is the output of a transaction whose + /// inputs are all from this wallet. For unconfirmed coins, we + /// further require that all unconfirmed ancestors, if any, also + /// be from self, as otherwise they will depend on an unconfirmed + /// external transaction. + pub is_from_self: bool, } impl TryFrom<&rusqlite::Row<'_>> for DbCoin { @@ -234,6 +242,7 @@ impl TryFrom<&rusqlite::Row<'_>> for DbCoin { }); let is_immature: bool = row.get(12)?; + let is_from_self: bool = row.get(13)?; Ok(DbCoin { id, @@ -246,6 +255,7 @@ impl TryFrom<&rusqlite::Row<'_>> for DbCoin { is_change, spend_txid, spend_block, + is_from_self, }) } } diff --git a/lianad/src/database/sqlite/utils.rs b/lianad/src/database/sqlite/utils.rs index 95fc984be..e967ea794 100644 --- a/lianad/src/database/sqlite/utils.rs +++ b/lianad/src/database/sqlite/utils.rs @@ -3,6 +3,7 @@ use crate::database::sqlite::{FreshDbOptions, SqliteDbError, DB_VERSION}; use std::{convert::TryInto, fs, path, time}; use miniscript::bitcoin::{self, secp256k1}; +use rusqlite::Transaction; pub const LOOK_AHEAD_LIMIT: u32 = 200; @@ -150,6 +151,74 @@ pub fn db_version(conn: &mut rusqlite::Connection) -> Result .expect("There is always a row in the version table")) } +/// Update `is_from_self` in coins table for all unconfirmed coins +/// and those confirmed after `current_tip_height`. +/// +/// This only sets the value to true as we do not expect the value +/// to change from true to false. In case of a reorg, the value +/// for all unconfirmed coins should be set to false before this +/// function is called. +pub fn update_coins_from_self( + db_tx: &Transaction, + current_tip_height: i32, +) -> Result<(), rusqlite::Error> { + // Given the requirement for unconfirmed coins that all ancestors + // be from self, we perform the update in a loop until no further + // rows are updated in order to iterate over the unconfirmed coins. + let mut i = 0; + loop { + i += 1; + // Although we don't expect any unconfirmed transaction to have + // more than 25 in-mempool descendants including itself, there + // could be more descendants in the DB following a reorg and a + // rollback of the tip. Therefore, we can't be sure how many + // iterations will be required here and can't set a max number + // of iterations. However, the query only sets `is_from_self` + // to 1 for those coins with value 0 and so the number of rows + // affected by each iteration must become 0, given there are a + // finite number of rows, at which point the loop will break. + if i % 5 == 0 { + log::debug!("Setting is_from_self. Starting iteration {}..", i); + } + let updated = db_tx.execute( + " + UPDATE coins + SET is_from_self = 1 + FROM transactions t + INNER JOIN ( + SELECT + spend_txid, + SUM( + CASE + WHEN blockheight IS NOT NULL THEN 1 + -- If the spending coin is unconfirmed, only count + -- it as an input coin if it is from self. + WHEN blockheight IS NULL AND is_from_self = 1 THEN 1 + ELSE 0 + END + ) AS cnt + FROM coins + WHERE spend_txid IS NOT NULL + -- We only need to consider spend transactions that are + -- unconfirmed or confirmed after `current_tip_height + -- as only these transactions will affect the coins that + -- we are updating. + AND (spend_block_height IS NULL OR spend_block_height > ?1) + GROUP BY spend_txid + ) spends + ON t.txid = spends.spend_txid AND t.num_inputs = spends.cnt + WHERE coins.txid = t.txid + AND (coins.blockheight IS NULL OR coins.blockheight > ?1) + AND coins.is_from_self = 0 + ", + [current_tip_height], + )?; + if updated == 0 { + break Ok(()); + } + } +} + // In Liana 0.4 we upgraded the schema to hold a timestamp for transaction drafts. Existing // transaction drafts are not set any timestamp on purpose. fn migrate_v0_to_v1(conn: &mut rusqlite::Connection) -> Result<(), SqliteDbError> { @@ -307,6 +376,329 @@ fn migrate_v5_to_v6(conn: &mut rusqlite::Connection) -> Result<(), SqliteDbError Ok(()) } +fn migrate_v6_to_v7(conn: &mut rusqlite::Connection) -> Result<(), SqliteDbError> { + db_exec(conn, |db_tx| { + let num_coins: i64 = + db_tx.query_row("SELECT count(*) AS cnt FROM coins", [], |row| row.get(0))?; + // Create a copy of the coins table without any FK constraints. + db_tx.execute_batch( + " + CREATE TABLE coins_copy ( + id INTEGER PRIMARY KEY NOT NULL, + wallet_id INTEGER NOT NULL, + blockheight INTEGER, + blocktime INTEGER, + txid BLOB NOT NULL, + vout INTEGER NOT NULL, + amount_sat INTEGER NOT NULL, + derivation_index INTEGER NOT NULL, + is_change BOOLEAN NOT NULL CHECK (is_change IN (0,1)), + spend_txid BLOB, + spend_block_height INTEGER, + spend_block_time INTEGER, + is_immature BOOLEAN NOT NULL CHECK (is_immature IN (0,1)), + UNIQUE (txid, vout) + ); + + INSERT INTO coins_copy ( + id, + wallet_id, + blockheight, + blocktime, + txid, + vout, + amount_sat, + derivation_index, + is_change, + spend_txid, + spend_block_height, + spend_block_time, + is_immature + ) SELECT + id, + wallet_id, + blockheight, + blocktime, + txid, + vout, + amount_sat, + derivation_index, + is_change, + spend_txid, + spend_block_height, + spend_block_time, + is_immature + FROM coins; + ", + )?; + + // Make sure all rows of coins_copy match the expected values. + let num_coins_intersect: i64 = db_tx.query_row( + "SELECT count(*) AS cnt FROM + ( + SELECT + id, + wallet_id, + blockheight, + blocktime, + txid, + vout, + amount_sat, + derivation_index, + is_change, + spend_txid, + spend_block_height, + spend_block_time, + is_immature + FROM + coins + + INTERSECT + + SELECT + id, + wallet_id, + blockheight, + blocktime, + txid, + vout, + amount_sat, + derivation_index, + is_change, + spend_txid, + spend_block_height, + spend_block_time, + is_immature + FROM + coins_copy + )", + [], + |row| row.get(0), + )?; + assert_eq!(num_coins, num_coins_intersect); + + let num_txs: i64 = + db_tx.query_row("SELECT count(*) AS cnt FROM transactions", [], |row| { + row.get(0) + })?; + // Rename transactions table and add the new `num_inputs` column as nullable. + db_tx.execute_batch( + " + ALTER TABLE transactions RENAME TO transactions_old; + ALTER TABLE transactions_old ADD COLUMN num_inputs INTEGER; + ", + )?; + + // Iterate over transactions, updating the `num_inputs` column. + let txids = db_tx + .prepare("SELECT txid FROM transactions_old")? + .query_map([], |row| { + let txid: Vec = row.get(0)?; + let txid: bitcoin::Txid = bitcoin::consensus::encode::deserialize(&txid) + .expect("We only store valid txids"); + Ok(txid) + })? + .collect::>>()?; + let num_txids: i64 = txids + .len() + .try_into() + .expect("Number of txids must be less than 2**63"); + assert_eq!(num_txids, num_txs); + for txid in &txids { + let tx = db_tx.query_row( + "SELECT tx FROM transactions_old WHERE txid = ?1", + rusqlite::params![txid[..].to_vec()], + |row| { + let tx: Vec = row.get(0)?; + let tx: bitcoin::Transaction = bitcoin::consensus::encode::deserialize(&tx) + .expect("We only store valid transactions"); + Ok(tx) + }, + )?; + let num_inputs: i64 = tx + .input + .len() + .try_into() + .expect("Num tx inputs must be less than 2**63"); + db_tx.execute( + "UPDATE transactions_old SET num_inputs = ?1 WHERE txid = ?2", + rusqlite::params![num_inputs, txid[..].to_vec()], + )?; + } + + // Create the new transactions table and copy over the data. + db_tx.execute_batch( + " + CREATE TABLE transactions ( + id INTEGER PRIMARY KEY NOT NULL, + txid BLOB UNIQUE NOT NULL, + tx BLOB UNIQUE NOT NULL, + num_inputs INTEGER NOT NULL + ); + + INSERT INTO transactions(id, txid, tx, num_inputs) + SELECT id, txid, tx, num_inputs + FROM transactions_old;", + )?; + // Make sure we have the expected values. + let num_txs_intersect: i64 = db_tx.query_row( + "SELECT count(*) AS cnt FROM + ( + SELECT id, txid, tx, num_inputs + FROM transactions + INTERSECT + SELECT id, txid, tx, num_inputs + FROM transactions_old + )", + [], + |row| row.get(0), + )?; + assert_eq!(num_txs, num_txs_intersect); + + // Create and populate the new coins table, + // setting `is_from_self` to 0 for all rows. + db_tx.execute_batch( + " + DROP TABLE coins; + DROP TABLE transactions_old; + + CREATE TABLE coins ( + id INTEGER PRIMARY KEY NOT NULL, + wallet_id INTEGER NOT NULL, + blockheight INTEGER, + blocktime INTEGER, + txid BLOB NOT NULL, + vout INTEGER NOT NULL, + amount_sat INTEGER NOT NULL, + derivation_index INTEGER NOT NULL, + is_change BOOLEAN NOT NULL CHECK (is_change IN (0,1)), + spend_txid BLOB, + spend_block_height INTEGER, + spend_block_time INTEGER, + is_immature BOOLEAN NOT NULL CHECK (is_immature IN (0,1)), + is_from_self BOOLEAN NOT NULL CHECK (is_from_self IN (0,1)), + UNIQUE (txid, vout), + FOREIGN KEY (wallet_id) REFERENCES wallets (id) + ON UPDATE RESTRICT + ON DELETE RESTRICT, + FOREIGN KEY (txid) REFERENCES transactions (txid) + ON UPDATE RESTRICT + ON DELETE RESTRICT, + FOREIGN KEY (spend_txid) REFERENCES transactions (txid) + ON UPDATE RESTRICT + ON DELETE RESTRICT + ); + + INSERT INTO coins ( + id, + wallet_id, + blockheight, + blocktime, + txid, + vout, + amount_sat, + derivation_index, + is_change, + spend_txid, + spend_block_height, + spend_block_time, + is_immature, + is_from_self + ) SELECT + id, + wallet_id, + blockheight, + blocktime, + txid, + vout, + amount_sat, + derivation_index, + is_change, + spend_txid, + spend_block_height, + spend_block_time, + is_immature, + 0 + FROM coins_copy; + ", + )?; + + // Make sure all rows in the newly created coins table are as expected. + let num_coins_intersect: i64 = db_tx.query_row( + "SELECT count(*) AS cnt FROM + ( + SELECT + id, + wallet_id, + blockheight, + blocktime, + txid, + vout, + amount_sat, + derivation_index, + is_change, + spend_txid, + spend_block_height, + spend_block_time, + is_immature, + is_from_self + FROM + coins + + INTERSECT + + SELECT + id, + wallet_id, + blockheight, + blocktime, + txid, + vout, + amount_sat, + derivation_index, + is_change, + spend_txid, + spend_block_height, + spend_block_time, + is_immature, + 0 -- we set all rows to 0 + FROM + coins_copy + )", + [], + |row| row.get(0), + )?; + assert_eq!(num_coins, num_coins_intersect); + + { + // Make sure there are no coins with confirmation height <= 0: + let row_count: i64 = db_tx.query_row( + "SELECT count(*) AS cnt FROM coins WHERE blockheight <= 0", + [], + |row| row.get(0), + )?; + assert_eq!(row_count, 0); + } + // Update the `is_from_self` column for all unconfirmed coins and those + // confirmed after height 0, i.e. this will act on all coins. + // For now, we use this utils function directly in the migration as we + // don't expect any changes to the DB schema that would stop this function + // working on a V6 DB schema. In future, this migration can use a "frozen" + // version of this function if required. + update_coins_from_self(db_tx, 0)?; + + db_tx.execute_batch( + " + DROP TABLE coins_copy; + + UPDATE version SET version = 7;", + )?; + + Ok(()) + })?; + Ok(()) +} + /// Check the database version and if necessary apply the migrations to upgrade it to the current /// one. The `bitcoin_txs` parameter is here for the migration from versions 4 and earlier, which /// did not store the Bitcoin transactions in database, to versions 5 and later, which do. For a @@ -360,6 +752,11 @@ pub fn maybe_apply_migration( migrate_v5_to_v6(&mut conn)?; log::warn!("Migration from database version 5 to version 6 successful."); } + 6 => { + log::warn!("Upgrading database from version 6 to version 7."); + migrate_v6_to_v7(&mut conn)?; + log::warn!("Migration from database version 6 to version 7 successful."); + } _ => return Err(SqliteDbError::UnsupportedVersion(version)), } } diff --git a/lianad/src/testutils.rs b/lianad/src/testutils.rs index 04ae68204..a7bbb2a1e 100644 --- a/lianad/src/testutils.rs +++ b/lianad/src/testutils.rs @@ -484,6 +484,10 @@ impl DatabaseConnection for DummyDatabase { } } + fn update_coins_from_self(&mut self, _current_tip_height: i32) { + // noop + } + fn list_wallet_transactions( &mut self, txids: &[bitcoin::Txid],