From e267f66be6ce3bebac413ee9e4901d27e8cbe9df Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Mon, 15 Jul 2024 17:58:56 +0100 Subject: [PATCH 01/17] descriptors: allow to get underlying public key --- src/descriptors/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/descriptors/mod.rs b/src/descriptors/mod.rs index 7ff6ac30d..818ecb19c 100644 --- a/src/descriptors/mod.rs +++ b/src/descriptors/mod.rs @@ -613,6 +613,13 @@ impl SinglePathLianaDesc { ), ) } + + /// Reference to the underlying `Descriptor` + pub fn as_descriptor_public_key( + &self, + ) -> &descriptor::Descriptor { + &self.0 + } } pub enum DescKeysOrigins { From 34b9a49328e7a3313ad6f4a0537ac5b349b54d76 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:07:07 +0100 Subject: [PATCH 02/17] config: add general bitcoin backend option --- src/config.rs | 13 +++++++++++-- src/lib.rs | 27 +++++++++++++++++++-------- src/testutils.rs | 2 +- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/config.rs b/src/config.rs index c0f24ac12..6312b9e46 100644 --- a/src/config.rs +++ b/src/config.rs @@ -85,6 +85,14 @@ fn default_daemon() -> bool { false } +/// Bitcoin backend config. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub enum BitcoinBackend { + /// Settings specific to bitcoind as the Bitcoin interface. + #[serde(rename = "bitcoind_config")] + Bitcoind(BitcoindConfig), +} + /// RPC authentication options. #[derive(Clone, PartialEq, Serialize)] pub enum BitcoindRpcAuth { @@ -152,8 +160,9 @@ pub struct Config { pub main_descriptor: LianaDescriptor, /// Settings for the Bitcoin interface pub bitcoin_config: BitcoinConfig, - /// Settings specific to bitcoind as the Bitcoin interface - pub bitcoind_config: Option, + /// Settings specific to the Bitcoin backend. + #[serde(flatten)] + pub bitcoin_backend: Option, } impl Config { diff --git a/src/lib.rs b/src/lib.rs index 584a3f3c7..8d811f6ab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -92,6 +92,7 @@ pub enum StartupError { DefaultDataDirNotFound, DatadirCreation(path::PathBuf, io::Error), MissingBitcoindConfig, + MissingBitcoinBackendConfig, DbMigrateBitcoinTxs(&'static str), Database(SqliteDbError), Bitcoind(BitcoindError), @@ -117,6 +118,10 @@ impl fmt::Display for StartupError { f, "Our Bitcoin interface is bitcoind but we have no 'bitcoind_config' entry in the configuration." ), + Self::MissingBitcoinBackendConfig => write!( + f, + "No Bitcoin backend entry in the configuration." + ), Self::DbMigrateBitcoinTxs(msg) => write!( f, "Error when migrating Bitcoin transaction from Bitcoin backend to database: {}.", msg @@ -260,8 +265,8 @@ fn setup_bitcoind( #[cfg(target_os = "windows")] let wo_path_str = wo_path_str.replace("\\\\?\\", "").replace("\\\\?", ""); - let bitcoind_config = config - .bitcoind_config + let config::BitcoinBackend::Bitcoind(bitcoind_config) = config + .bitcoin_backend .as_ref() .ok_or(StartupError::MissingBitcoindConfig)?; let bitcoind = BitcoinD::new(bitcoind_config, wo_path_str)?; @@ -374,7 +379,11 @@ impl DaemonHandle { // Set up the connection to bitcoind (if using it) first as we may need it for the database // migration when setting up SQLite below. let bitcoind = if bitcoin.is_none() { - Some(setup_bitcoind(&config, &data_dir, fresh_data_dir)?) + if let Some(config::BitcoinBackend::Bitcoind(_)) = &config.bitcoin_backend { + Some(setup_bitcoind(&config, &data_dir, fresh_data_dir)?) + } else { + None + } } else { None }; @@ -392,11 +401,13 @@ impl DaemonHandle { }; // Finally set up the Bitcoin backend. - let bit = match (bitcoin, bitcoind) { - (Some(bit), None) => sync::Arc::from(sync::Mutex::from(bit)), - (None, Some(bit)) => sync::Arc::from(sync::Mutex::from(bit)) + let bit = match (bitcoin, &config.bitcoin_backend) { + (Some(bit), _) => sync::Arc::from(sync::Mutex::from(bit)), + (None, Some(config::BitcoinBackend::Bitcoind(..))) => sync::Arc::from( + sync::Mutex::from(bitcoind.expect("bitcoind must have been set already")), + ) as sync::Arc>, - _ => unreachable!("Either bitcoind or bitcoin interface is always set."), + (None, None) => Err(StartupError::MissingBitcoinBackendConfig)?, }; // If we are on a UNIX system and they told us to daemonize, do it now. @@ -764,7 +775,7 @@ mod tests { let change_desc = desc.change_descriptor().clone(); let config = Config { bitcoin_config, - bitcoind_config: Some(bitcoind_config), + bitcoin_backend: Some(config::BitcoinBackend::Bitcoind(bitcoind_config)), data_dir: Some(data_dir), #[cfg(unix)] daemon: false, diff --git a/src/testutils.rs b/src/testutils.rs index 3669b5fc0..cb3418f9b 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -534,7 +534,7 @@ impl DummyLiana { let desc = descriptors::LianaDescriptor::new(policy); let config = Config { bitcoin_config, - bitcoind_config: None, + bitcoin_backend: None, data_dir: Some(data_dir), #[cfg(unix)] daemon: false, From 89e004d4fdd88324da255bc85aeb8adbb90653b1 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:55:05 +0100 Subject: [PATCH 03/17] bitcoin: add sync_wallet method to interface This includes changes from darosior's comment in https://github.com/wizardsardine/liana/pull/1222#issuecomment-2324894986. --- src/bitcoin/mod.rs | 32 +++++++++++++++++++++++++++++++- src/testutils.rs | 8 ++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/bitcoin/mod.rs b/src/bitcoin/mod.rs index a278d33aa..effc03200 100644 --- a/src/bitcoin/mod.rs +++ b/src/bitcoin/mod.rs @@ -13,7 +13,7 @@ pub use d::{MempoolEntry, SyncProgress}; use std::{fmt, sync}; -use miniscript::bitcoin::{self, address}; +use miniscript::bitcoin::{self, address, bip32::ChildNumber}; const COINBASE_MATURITY: i32 = 100; @@ -58,6 +58,17 @@ pub trait BitcoinInterface: Send { /// Check whether this former tip is part of the current best chain. fn is_in_chain(&self, tip: &BlockChainTip) -> bool; + /// Sync the wallet with the current best chain. + /// `receive_index` and `change_index` are the last derivation indices + /// that are expected to have been used by the wallet. + /// In case there has been a reorg, returns the common ancestor between + /// the wallet and the reorged chain. + fn sync_wallet( + &mut self, + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Result, String>; + /// Get coins received since the specified tip. fn received_coins( &self, @@ -157,6 +168,15 @@ impl BitcoinInterface for d::BitcoinD { .unwrap_or(false) } + // The watchonly wallet handles this for us. + fn sync_wallet( + &mut self, + _receive_index: ChildNumber, + _change_index: ChildNumber, + ) -> Result, String> { + Ok(None) + } + fn received_coins( &self, tip: &BlockChainTip, @@ -410,6 +430,16 @@ impl BitcoinInterface for sync::Arc> self.lock().unwrap().is_in_chain(tip) } + fn sync_wallet( + &mut self, + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Result, String> { + self.lock() + .unwrap() + .sync_wallet(receive_index, change_index) + } + fn received_coins( &self, tip: &BlockChainTip, diff --git a/src/testutils.rs b/src/testutils.rs index cb3418f9b..ffea3ddc9 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -64,6 +64,14 @@ impl BitcoinInterface for DummyBitcoind { true } + fn sync_wallet( + &mut self, + _receive_index: bip32::ChildNumber, + _change_index: bip32::ChildNumber, + ) -> Result, String> { + Ok(None) + } + fn received_coins( &self, _: &BlockChainTip, From 69259c12362b2be8da8f210def1d3e338fa8ab00 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Tue, 6 Aug 2024 16:25:50 +0100 Subject: [PATCH 04/17] poller: sync wallet before checking updates This includes changes from darosior's comment in https://github.com/wizardsardine/liana/pull/1222#issuecomment-2324894986. --- src/bitcoin/poller/looper.rs | 46 +++++++++++++++++++++++++++--------- src/bitcoin/poller/mod.rs | 6 ++--- src/lib.rs | 2 +- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/bitcoin/poller/looper.rs b/src/bitcoin/poller/looper.rs index 8ca1c73ea..877bc9db8 100644 --- a/src/bitcoin/poller/looper.rs +++ b/src/bitcoin/poller/looper.rs @@ -4,7 +4,7 @@ use crate::{ descriptors, }; -use std::{collections::HashSet, sync, time}; +use std::{collections::HashSet, sync, thread, time}; use miniscript::bitcoin::{self, secp256k1}; @@ -239,20 +239,44 @@ fn new_tip(bit: &impl BitcoinInterface, current_tip: &BlockChainTip) -> TipUpdat fn updates( db_conn: &mut Box, - bit: &impl BitcoinInterface, + bit: &mut impl BitcoinInterface, descs: &[descriptors::SinglePathLianaDesc], secp: &secp256k1::Secp256k1, ) { - // Check if there was a new block before updating ourselves. + // Check if there was a new block before we update our state. + // + // Some backends (such as Electrum) need to perform an explicit sync to provide updated data + // about the Bitcoin network. For those the common ancestor is immediately returned in case + // there was a reorg. For other backends (such as bitcoind) this function always return + // `Ok(None)`. We leverage this to query the next tip and poll for reorgs only in this case. + // FIXME: harmonize the Bitcoin backend interface, this intricacy is due to the introduction of + // an Electrum backend with the bitcoind-specific backend interface. let current_tip = db_conn.chain_tip().expect("Always set at first startup"); - let latest_tip = match new_tip(bit, ¤t_tip) { - TipUpdate::Same => current_tip, - TipUpdate::Progress(new_tip) => new_tip, - TipUpdate::Reorged(new_tip) => { + let (receive_index, change_index) = (db_conn.receive_index(), db_conn.change_index()); + let latest_tip = match bit.sync_wallet(receive_index, change_index) { + Ok(None) => { + match new_tip(bit, ¤t_tip) { + TipUpdate::Same => current_tip, + TipUpdate::Progress(new_tip) => new_tip, + TipUpdate::Reorged(new_tip) => { + // The block chain was reorganized. Rollback our state down to the common ancestor + // between our former chain and the new one, then restart fresh. + db_conn.rollback_tip(&new_tip); + log::info!("Tip was rolled back to '{}'.", new_tip); + return updates(db_conn, bit, descs, secp); + } + } + } + Ok(Some(reorg_common_ancestor)) => { // The block chain was reorganized. Rollback our state down to the common ancestor // between our former chain and the new one, then restart fresh. - db_conn.rollback_tip(&new_tip); - log::info!("Tip was rolled back to '{}'.", new_tip); + db_conn.rollback_tip(&reorg_common_ancestor); + log::info!("Tip was rolled back to '{}'.", &reorg_common_ancestor); + return updates(db_conn, bit, descs, secp); + } + Err(e) => { + log::error!("Error syncing wallet: '{}'.", e); + thread::sleep(time::Duration::from_secs(2)); return updates(db_conn, bit, descs, secp); } }; @@ -289,7 +313,7 @@ fn updates( // Check if there is any rescan of the backend ongoing or one that just finished. fn rescan_check( db_conn: &mut Box, - bit: &impl BitcoinInterface, + bit: &mut impl BitcoinInterface, descs: &[descriptors::SinglePathLianaDesc], secp: &secp256k1::Secp256k1, ) { @@ -356,7 +380,7 @@ pub fn sync_poll_interval() -> time::Duration { /// Update our state from the Bitcoin backend. pub fn poll( - bit: &sync::Arc>, + bit: &mut sync::Arc>, db: &sync::Arc>, secp: &secp256k1::Secp256k1, descs: &[descriptors::SinglePathLianaDesc], diff --git a/src/bitcoin/poller/mod.rs b/src/bitcoin/poller/mod.rs index 8e9874e16..c2986238e 100644 --- a/src/bitcoin/poller/mod.rs +++ b/src/bitcoin/poller/mod.rs @@ -56,7 +56,7 @@ impl Poller { /// Typically this would run for the whole duration of the program in a thread, and the main /// thread would set the `shutdown` atomic to `true` when shutting down. pub fn poll_forever( - &self, + &mut self, poll_interval: time::Duration, receiver: mpsc::Receiver, ) { @@ -91,7 +91,7 @@ impl Poller { // We've been asked to poll, don't wait any further and signal completion to // the caller. last_poll = Some(time::Instant::now()); - looper::poll(&self.bit, &self.db, &self.secp, &self.descs); + looper::poll(&mut self.bit, &self.db, &self.secp, &self.descs); if let Err(e) = sender.send(()) { log::error!("Error sending immediate poll completion signal: {}.", e); } @@ -122,7 +122,7 @@ impl Poller { } } - looper::poll(&self.bit, &self.db, &self.secp, &self.descs); + looper::poll(&mut self.bit, &self.db, &self.secp, &self.descs); } } } diff --git a/src/lib.rs b/src/lib.rs index 8d811f6ab..fa30706ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -426,7 +426,7 @@ impl DaemonHandle { // Start the poller thread. Keep the thread handle to be able to check if it crashed. Store // an atomic to be able to stop it. - let bitcoin_poller = + let mut bitcoin_poller = poller::Poller::new(bit.clone(), db.clone(), config.main_descriptor.clone()); let (poller_sender, poller_receiver) = mpsc::sync_channel(0); let poller_handle = thread::Builder::new() From 4c02b0d43115f5d792908dd30580eaf241e8fc58 Mon Sep 17 00:00:00 2001 From: Michael Mallan Date: Wed, 4 Sep 2024 10:53:29 +0100 Subject: [PATCH 05/17] bitcoin: use mut ref for start_rescan --- src/bitcoin/d/mod.rs | 2 +- src/bitcoin/mod.rs | 6 +++--- src/commands/mod.rs | 2 +- src/jsonrpc/api.rs | 4 ++-- src/jsonrpc/server.rs | 4 ++-- src/testutils.rs | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/bitcoin/d/mod.rs b/src/bitcoin/d/mod.rs index 41e25216a..fe26b7327 100644 --- a/src/bitcoin/d/mod.rs +++ b/src/bitcoin/d/mod.rs @@ -1096,7 +1096,7 @@ impl BitcoinD { } pub fn start_rescan( - &self, + &mut self, desc: &LianaDescriptor, timestamp: u32, ) -> Result<(), BitcoindError> { diff --git a/src/bitcoin/mod.rs b/src/bitcoin/mod.rs index effc03200..6e47f1487 100644 --- a/src/bitcoin/mod.rs +++ b/src/bitcoin/mod.rs @@ -109,7 +109,7 @@ pub trait BitcoinInterface: Send { /// Trigger a rescan of the block chain for transactions related to this descriptor since /// the given date. fn start_rescan( - &self, + &mut self, desc: &descriptors::LianaDescriptor, timestamp: u32, ) -> Result<(), String>; @@ -367,7 +367,7 @@ impl BitcoinInterface for d::BitcoinD { } fn start_rescan( - &self, + &mut self, desc: &descriptors::LianaDescriptor, timestamp: u32, ) -> Result<(), String> { @@ -481,7 +481,7 @@ impl BitcoinInterface for sync::Arc> } fn start_rescan( - &self, + &mut self, desc: &descriptors::LianaDescriptor, timestamp: u32, ) -> Result<(), String> { diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 6e2b20636..092e87e74 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -986,7 +986,7 @@ impl DaemonControl { /// Trigger a rescan of the block chain for transactions involving our main descriptor between /// the given date and the current tip. /// The date must be after the genesis block time and before the current tip blocktime. - pub fn start_rescan(&self, timestamp: u32) -> Result<(), CommandError> { + pub fn start_rescan(&mut self, timestamp: u32) -> Result<(), CommandError> { let mut db_conn = self.db.connection(); let genesis_timestamp = self.bitcoin.genesis_block_timestamp(); diff --git a/src/jsonrpc/api.rs b/src/jsonrpc/api.rs index 5835e2e9c..3ff137afb 100644 --- a/src/jsonrpc/api.rs +++ b/src/jsonrpc/api.rs @@ -266,7 +266,7 @@ fn list_transactions(control: &DaemonControl, params: Params) -> Result Result { +fn start_rescan(control: &mut DaemonControl, params: Params) -> Result { let timestamp: u32 = params .get(0, "timestamp") .ok_or_else(|| Error::invalid_params("Missing 'timestamp' parameter."))? @@ -365,7 +365,7 @@ fn get_labels(control: &DaemonControl, params: Params) -> Result Result { +pub fn handle_request(control: &mut DaemonControl, req: Request) -> Result { let result = match req.method.as_str() { "broadcastspend" => { let params = req diff --git a/src/jsonrpc/server.rs b/src/jsonrpc/server.rs index 424adc2a0..49b0d925a 100644 --- a/src/jsonrpc/server.rs +++ b/src/jsonrpc/server.rs @@ -81,7 +81,7 @@ fn read_command( // Handle all messages from this connection. fn connection_handler( - control: DaemonControl, + mut control: DaemonControl, mut stream: net::UnixStream, shutdown: sync::Arc, ) -> Result<(), io::Error> { @@ -106,7 +106,7 @@ fn connection_handler( log::trace!("JSONRPC request: {:?}", serde_json::to_string(&req)); let response = - api::handle_request(&control, req).unwrap_or_else(|e| Response::error(req_id, e)); + api::handle_request(&mut control, req).unwrap_or_else(|e| Response::error(req_id, e)); log::trace!("JSONRPC response: {:?}", serde_json::to_string(&response)); if let Err(e) = serde_json::to_writer(&stream, &response) { log::error!("Error writing response: '{}'", e); diff --git a/src/testutils.rs b/src/testutils.rs index ffea3ddc9..c5800b90b 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -109,7 +109,7 @@ impl BitcoinInterface for DummyBitcoind { todo!() } - fn start_rescan(&self, _: &descriptors::LianaDescriptor, _: u32) -> Result<(), String> { + fn start_rescan(&mut self, _: &descriptors::LianaDescriptor, _: u32) -> Result<(), String> { todo!() } From 689442c2c910ad5967651c0cc9f84bdc71cd0002 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Tue, 6 Aug 2024 16:27:05 +0100 Subject: [PATCH 06/17] bitcoin: allow to store UTXO deriv index --- src/bitcoin/mod.rs | 12 +++++- src/bitcoin/poller/looper.rs | 81 ++++++++++++++++++++---------------- 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/bitcoin/mod.rs b/src/bitcoin/mod.rs index 6e47f1487..bd9c67351 100644 --- a/src/bitcoin/mod.rs +++ b/src/bitcoin/mod.rs @@ -204,7 +204,7 @@ impl BitcoinInterface for d::BitcoinD { outpoint, amount, block_height, - address, + address: UTxOAddress::Address(address), is_immature, }) } else { @@ -523,6 +523,14 @@ pub struct UTxO { pub outpoint: bitcoin::OutPoint, pub amount: bitcoin::Amount, pub block_height: Option, - pub address: bitcoin::Address, + pub address: UTxOAddress, pub is_immature: bool, } + +/// Details about the UTXO address. +#[derive(Debug, Clone)] +pub enum UTxOAddress { + Address(bitcoin::Address), + /// Derivation index and whether it is from the change descriptor. + DerivIndex(ChildNumber, bool), +} diff --git a/src/bitcoin/poller/looper.rs b/src/bitcoin/poller/looper.rs index 877bc9db8..39e6a6d69 100644 --- a/src/bitcoin/poller/looper.rs +++ b/src/bitcoin/poller/looper.rs @@ -1,5 +1,5 @@ use crate::{ - bitcoin::{BitcoinInterface, BlockChainTip, UTxO}, + bitcoin::{BitcoinInterface, BlockChainTip, UTxO, UTxOAddress}, database::{Coin, DatabaseConnection, DatabaseInterface}, descriptors, }; @@ -46,44 +46,53 @@ fn update_coins( .. } = utxo; // We can only really treat them if we know the derivation index that was used. - let address = match address.require_network(network) { - Ok(addr) => addr, - Err(e) => { - log::error!("Invalid network for address: {}", e); - continue; + let (derivation_index, is_change) = match address { + UTxOAddress::Address(address) => { + let address = match address.require_network(network) { + Ok(addr) => addr, + Err(e) => { + log::error!("Invalid network for address: {}", e); + continue; + } + }; + if let Some((derivation_index, is_change)) = + db_conn.derivation_index_by_address(&address) + { + (derivation_index, is_change) + } else { + // TODO: maybe we could try out something here? Like bruteforcing the next 200 indexes? + log::error!( + "Could not get derivation index for coin '{}' (address: '{}')", + &utxo.outpoint, + &address + ); + continue; + } } + UTxOAddress::DerivIndex(index, is_change) => (index, is_change), }; - if let Some((derivation_index, is_change)) = db_conn.derivation_index_by_address(&address) { - // First of if we are receiving coins that are beyond our next derivation index, - // adjust it. - if derivation_index > db_conn.receive_index() { - db_conn.set_receive_index(derivation_index, secp); - } - if derivation_index > db_conn.change_index() { - db_conn.set_change_index(derivation_index, secp); - } + // First of if we are receiving coins that are beyond our next derivation index, + // adjust it. + if derivation_index > db_conn.receive_index() { + db_conn.set_receive_index(derivation_index, secp); + } + if derivation_index > db_conn.change_index() { + db_conn.set_change_index(derivation_index, secp); + } - // Now record this coin as a newly received one. - if !curr_coins.contains_key(&utxo.outpoint) { - let coin = Coin { - outpoint, - is_immature, - amount, - derivation_index, - is_change, - block_info: None, - spend_txid: None, - spend_block: None, - }; - received.push(coin); - } - } else { - // TODO: maybe we could try out something here? Like bruteforcing the next 200 indexes? - log::error!( - "Could not get derivation index for coin '{}' (address: '{}')", - &utxo.outpoint, - &address - ); + // Now record this coin as a newly received one. + if !curr_coins.contains_key(&utxo.outpoint) { + let coin = Coin { + outpoint, + is_immature, + amount, + derivation_index, + is_change, + block_info: None, + spend_txid: None, + spend_block: None, + }; + received.push(coin); } } log::debug!("Newly received coins: {:?}", received); From 5011ad929037af294dc471844d8bf5c799040383 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Tue, 6 Aug 2024 17:25:46 +0100 Subject: [PATCH 07/17] bitcoin: return spent block height & time separately --- src/bitcoin/mod.rs | 22 ++++++++-------------- src/bitcoin/poller/looper.rs | 4 ---- src/testutils.rs | 2 +- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/bitcoin/mod.rs b/src/bitcoin/mod.rs index bd9c67351..6710434ec 100644 --- a/src/bitcoin/mod.rs +++ b/src/bitcoin/mod.rs @@ -15,6 +15,9 @@ use std::{fmt, sync}; use miniscript::bitcoin::{self, address, bip32::ChildNumber}; +// A spent coin's outpoint together with its spend transaction's txid, height and time. +type SpentCoin = (bitcoin::OutPoint, bitcoin::Txid, i32, u32); + const COINBASE_MATURITY: i32 = 100; /// Information about a block @@ -95,10 +98,7 @@ pub trait BitcoinInterface: Send { fn spent_coins( &self, outpoints: &[(bitcoin::OutPoint, bitcoin::Txid)], - ) -> ( - Vec<(bitcoin::OutPoint, bitcoin::Txid, Block)>, - Vec, - ); + ) -> (Vec, Vec); /// Get the common ancestor between the Bitcoin backend's tip and the given tip. fn common_ancestor(&self, tip: &BlockChainTip) -> Option; @@ -281,10 +281,7 @@ impl BitcoinInterface for d::BitcoinD { fn spent_coins( &self, outpoints: &[(bitcoin::OutPoint, bitcoin::Txid)], - ) -> ( - Vec<(bitcoin::OutPoint, bitcoin::Txid, Block)>, - Vec, - ) { + ) -> (Vec, Vec) { // Spend coins to be returned. let mut spent = Vec::with_capacity(outpoints.len()); // Coins whose spending transaction isn't in our local mempool anymore. @@ -302,7 +299,7 @@ impl BitcoinInterface for d::BitcoinD { // If the transaction was confirmed, mark it as such. if let Some(block) = res.block { - spent.push((*op, *txid, block)); + spent.push((*op, *txid, block.height, block.time)); continue; } @@ -325,7 +322,7 @@ impl BitcoinInterface for d::BitcoinD { }) }); if let Some((txid, block)) = conflict { - spent.push((*op, txid, block)); + spent.push((*op, txid, block.height, block.time)); continue; } @@ -465,10 +462,7 @@ impl BitcoinInterface for sync::Arc> fn spent_coins( &self, outpoints: &[(bitcoin::OutPoint, bitcoin::Txid)], - ) -> ( - Vec<(bitcoin::OutPoint, bitcoin::Txid, Block)>, - Vec, - ) { + ) -> (Vec, Vec) { self.lock().unwrap().spent_coins(outpoints) } diff --git a/src/bitcoin/poller/looper.rs b/src/bitcoin/poller/looper.rs index 39e6a6d69..20cc1d1b6 100644 --- a/src/bitcoin/poller/looper.rs +++ b/src/bitcoin/poller/looper.rs @@ -149,10 +149,6 @@ fn update_coins( .chain(spending.iter().cloned()) .collect(); let (spent, expired_spending) = bit.spent_coins(spending_coins.as_slice()); - let spent = spent - .into_iter() - .map(|(oupoint, txid, block)| (oupoint, txid, block.height, block.time)) - .collect(); log::debug!("Newly spent coins: {:?}", spent); UpdatedCoins { diff --git a/src/testutils.rs b/src/testutils.rs index c5800b90b..a5642b70f 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -95,7 +95,7 @@ impl BitcoinInterface for DummyBitcoind { &self, _: &[(bitcoin::OutPoint, bitcoin::Txid)], ) -> ( - Vec<(bitcoin::OutPoint, bitcoin::Txid, Block)>, + Vec<(bitcoin::OutPoint, bitcoin::Txid, i32, u32)>, Vec, ) { (Vec::new(), Vec::new()) From c4c2424f244fcd4461cf5378f83e9abe4ba12866 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:41:10 +0100 Subject: [PATCH 08/17] bitcoin: expose MempoolEntryFees --- src/bitcoin/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bitcoin/mod.rs b/src/bitcoin/mod.rs index 6710434ec..9ddc8a98c 100644 --- a/src/bitcoin/mod.rs +++ b/src/bitcoin/mod.rs @@ -9,7 +9,7 @@ use crate::{ bitcoin::d::{BitcoindError, CachedTxGetter, LSBlockEntry}, descriptors, }; -pub use d::{MempoolEntry, SyncProgress}; +pub use d::{MempoolEntry, MempoolEntryFees, SyncProgress}; use std::{fmt, sync}; From c7ee8620204e72bcf990a994c1bdf89652f40c84 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:12:27 +0100 Subject: [PATCH 09/17] bitcoin: add electrum backend This includes changes from darosior's comment in https://github.com/wizardsardine/liana/pull/1222#issuecomment-2324894986. --- Cargo.lock | 222 +++++++++++++++++- Cargo.toml | 4 + src/bitcoin/electrum/client.rs | 411 +++++++++++++++++++++++++++++++++ src/bitcoin/electrum/mod.rs | 250 ++++++++++++++++++++ src/bitcoin/electrum/utils.rs | 55 +++++ src/bitcoin/electrum/wallet.rs | 328 ++++++++++++++++++++++++++ src/bitcoin/mod.rs | 206 +++++++++++++++++ src/config.rs | 12 + src/lib.rs | 90 +++++++- 9 files changed, 1563 insertions(+), 15 deletions(-) create mode 100644 src/bitcoin/electrum/client.rs create mode 100644 src/bitcoin/electrum/mod.rs create mode 100644 src/bitcoin/electrum/utils.rs create mode 100644 src/bitcoin/electrum/wallet.rs diff --git a/Cargo.lock b/Cargo.lock index 335a18d52..91059d2b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,12 +62,32 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +[[package]] +name = "bdk_chain" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c601c4dc7e6c3efa538a0afbb43b964cefab9a9b5e8f352fa0ca38145448a5e7" +dependencies = [ + "bitcoin", + "miniscript", +] + [[package]] name = "bdk_coin_select" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c084bf76f0f67546fc814ffa82044144be1bb4618183a15016c162f8b087ad4" +[[package]] +name = "bdk_electrum" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28906275aeb1f71dc32045670f06c8a26fb17cc62151a99f7425d258f4bda589" +dependencies = [ + "bdk_chain", + "electrum-client", +] + [[package]] name = "bech32" version = "0.10.0-beta" @@ -139,6 +159,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cc" version = "1.0.83" @@ -172,7 +198,24 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "electrum-client" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89008f106be6f303695522f2f4c1f28b40c3e8367ed8b3bb227f1f882cb52cc2" +dependencies = [ + "bitcoin", + "byteorder", + "libc", + "log", + "rustls", + "serde", + "serde_json", + "webpki-roots", + "winapi", ] [[package]] @@ -268,6 +311,7 @@ version = "6.0.0" dependencies = [ "backtrace", "bdk_coin_select", + "bdk_electrum", "bip39", "dirs", "fern", @@ -438,6 +482,21 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rusqlite" version = "0.30.0" @@ -458,12 +517,44 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "secp256k1" version = "0.28.0" @@ -521,6 +612,12 @@ version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "syn" version = "2.0.46" @@ -591,6 +688,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "vcpkg" version = "0.2.15" @@ -609,13 +712,50 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -624,13 +764,29 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -639,42 +795,90 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "zerocopy" version = "0.7.32" diff --git a/Cargo.toml b/Cargo.toml index b3ce9f770..31fc8f63a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,10 @@ miniscript = { version = "11.0", features = ["serde", "compiler", "base64"] } # Coin selection algorithms for spend transaction creation. bdk_coin_select = "0.3" +# For Electrum backend. This is the latest version with the same bitcoin version as +# the miniscript dependency. +bdk_electrum = { version = "0.14" } + # Don't reinvent the wheel dirs = "5.0" diff --git a/src/bitcoin/electrum/client.rs b/src/bitcoin/electrum/client.rs new file mode 100644 index 000000000..230adcf6e --- /dev/null +++ b/src/bitcoin/electrum/client.rs @@ -0,0 +1,411 @@ +use std::{collections::HashSet, convert::TryInto}; + +use bdk_electrum::{ + bdk_chain::{ + bitcoin, + local_chain::{CheckPoint, LocalChain}, + spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult}, + BlockId, ChainPosition, ConfirmationHeightAnchor, TxGraph, + }, + electrum_client::{self, Config, ElectrumApi}, + ElectrumExt, +}; + +use super::utils::{ + block_id_from_tip, height_i32_from_usize, height_usize_from_i32, outpoints_from_tx, +}; +use crate::{ + bitcoin::{electrum::utils::tip_from_block_id, BlockChainTip, MempoolEntry, MempoolEntryFees}, + config, +}; + +// Default batch size to use when making requests to the Electrum server. +const DEFAULT_BATCH_SIZE: usize = 200; + +// If Electrum takes more than 3 minutes to answer one of our queries, fail. +const RPC_SOCKET_TIMEOUT: u8 = 180; + +// Number of retries while communicating with the Electrum server. +// A retry happens with exponential back-off (base 2) so this makes us give up after (1+2+4+8+16+32=) 63 seconds. +const RETRY_LIMIT: u8 = 6; + +/// An error in the Electrum client. +#[derive(Debug)] +pub enum Error { + Server(electrum_client::Error), + TipChanged(BlockId, BlockId), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Error::Server(e) => write!(f, "Electrum error: '{}'.", e), + Error::TipChanged(expected, actual) => write!( + f, + "Electrum error: Expected tip '{}' but actual tip was {}.", + tip_from_block_id(*expected), + tip_from_block_id(*actual), + ), + } + } +} + +pub struct Client(electrum_client::Client); + +impl Client { + /// Create a new client and perform sanity checks. + pub fn new(electrum_config: &config::ElectrumConfig) -> Result { + let config = Config::builder() + .retry(RETRY_LIMIT) + .timeout(Some(RPC_SOCKET_TIMEOUT)) + .build(); + let client = + bdk_electrum::electrum_client::Client::from_config(&electrum_config.addr, config) + .map_err(Error::Server)?; + Ok(Self(client)) + } + + pub fn chain_tip(&self) -> Result { + self.0 + .block_headers_subscribe() + .map_err(Error::Server) + .map(|notif| BlockChainTip { + height: height_i32_from_usize(notif.height), + hash: notif.header.block_hash(), + }) + } + + fn genesis_block_header(&self) -> Result { + self.0.block_header(0).map_err(Error::Server) + } + + pub fn genesis_block_timestamp(&self) -> Result { + self.genesis_block_header().map(|header| header.time) + } + + pub fn genesis_block(&self) -> Result { + self.genesis_block_header().map(|header| BlockChainTip { + hash: header.block_hash(), + height: 0, + }) + } + + pub fn broadcast_tx(&self, tx: &bitcoin::Transaction) -> Result { + self.0.transaction_broadcast(tx).map_err(Error::Server) + } + + pub fn tip_time(&self) -> Result { + let tip_height = self.chain_tip()?.height; + self.0 + .block_header(height_usize_from_i32(tip_height)) + .map_err(Error::Server) + .map(|bh| bh.time) + } + + fn sync_with_confirmation_height_anchor( + &self, + request: SyncRequest, + fetch_prev_txouts: bool, + ) -> Result, Error> { + Ok(self + .0 + .sync(request, DEFAULT_BATCH_SIZE, fetch_prev_txouts) + .map_err(Error::Server)? + .with_confirmation_height_anchor()) + } + + /// Perform the given `SyncRequest` with `ConfirmationTimeHeightAnchor`. + pub fn sync_with_confirmation_time_height_anchor( + &self, + request: SyncRequest, + fetch_prev_txouts: bool, + ) -> Result { + self.0 + .sync(request, DEFAULT_BATCH_SIZE, fetch_prev_txouts) + .map_err(Error::Server)? + .with_confirmation_time_height_anchor(&self.0) + .map_err(Error::Server) + } + + /// Perform the given `FullScanRequest` with `ConfirmationTimeHeightAnchor`. + pub fn full_scan_with_confirmation_time_height_anchor( + &self, + request: FullScanRequest, + stop_gap: usize, + fetch_prev_txouts: bool, + ) -> Result, Error> { + self.0 + .full_scan(request, stop_gap, DEFAULT_BATCH_SIZE, fetch_prev_txouts) + .map_err(Error::Server)? + .with_confirmation_time_height_anchor(&self.0) + .map_err(Error::Server) + } + + /// Get mempool entries. + /// + /// If `expected_tip` is specified, the function will return `Error::TipChanged` if the chain tip + /// changes while the entries are being found. Otherwise, the function will restart in case the + /// chain tip changes before completion. + fn mempool_entries( + &self, + txids: HashSet, + expected_tip: Option, + ) -> Result, Error> { + log::debug!("Getting mempool entries for txids '{:?}'.", txids); + let mut graph = TxGraph::default(); + let mut local_chain = LocalChain::from_genesis_hash(self.genesis_block()?.hash).0; + let tip_block = if let Some(ref expected_tip) = expected_tip { + expected_tip.block_id() + } else { + block_id_from_tip(self.chain_tip()?) + }; + if tip_block.height > 0 { + let _ = local_chain + .insert_block(tip_block) + .expect("only contains genesis block"); + } + // First, get the tx itself and check it's unconfirmed. + let request = SyncRequest::from_chain_tip(local_chain.tip()).chain_txids(txids.clone()); + // We'll get prev txouts for this tx when we find its ancestors below. + let sync_result = self.sync_with_confirmation_height_anchor(request, false)?; + let _ = local_chain.apply_update(sync_result.chain_update); + // Store local tip after first sync. This will be our reference tip. + let local_tip = local_chain.tip(); + if let Some(ref expected_tip) = expected_tip { + if expected_tip != &local_chain.tip() { + return Err(Error::TipChanged( + expected_tip.block_id(), + local_chain.tip().block_id(), + )); + } + } + let mut desc_ops = Vec::new(); + let mut txs = Vec::new(); + for txid in &txids { + if let Some(ChainPosition::Unconfirmed(_)) = sync_result + .graph_update + .get_chain_position(&local_chain, local_chain.tip().block_id(), *txid) + { + let tx = sync_result + .graph_update + .get_tx(*txid) + .expect("we must have tx in graph after sync"); + desc_ops.extend(outpoints_from_tx(&tx)); + txs.push(tx); + } + } + let _ = graph.apply_update(sync_result.graph_update); + // Now iterate over increasing depths of descendants. + // As they are descendants, we can assume they are all unconfirmed. + while !desc_ops.is_empty() { + log::debug!("Syncing descendant outpoints: {:?}", desc_ops); + let request = SyncRequest::from_chain_tip(local_chain.tip()) + .cache_graph_txs(&graph) + .chain_outpoints(desc_ops.clone()); + // Fetch prev txouts to ensure we have all required txs in the graph to calculate fees. + // An unconfirmed descendant may have a confirmed parent that we wouldn't have in our graph. + let sync_result = self.sync_with_confirmation_height_anchor(request, true)?; + let _ = local_chain.apply_update(sync_result.chain_update); + if let Some(ref expected_tip) = expected_tip { + if expected_tip != &local_chain.tip() { + return Err(Error::TipChanged( + expected_tip.block_id(), + local_chain.tip().block_id(), + )); + } + } + if local_chain.tip() != local_tip { + log::debug!("Chain tip changed while getting mempool entry. Restarting."); + return self.mempool_entries(txids, expected_tip.clone()); + } + let _ = graph.apply_update(sync_result.graph_update); + // Get any txids spending the outpoints we've just synced against. + let desc_txids: HashSet<_> = graph + .filter_chain_txouts( + &local_chain, + local_chain.tip().block_id(), + desc_ops.iter().map(|op| ((), *op)), + ) + .filter_map(|(_, txout)| txout.spent_by.map(|(_, spend_txid)| spend_txid)) + .collect(); + desc_ops = desc_txids + .iter() + .flat_map(|txid| { + let desc_tx = graph + .get_tx(*txid) + .expect("we must have tx in graph after sync"); + outpoints_from_tx(&desc_tx) + }) + .collect(); + } + + // For each unconfirmed transaction, starting with `txid`, get its direct ancestors, which may be confirmed or unconfirmed. + // Continue until there are no more unconfirmed ancestors. + // Confirmed transactions will be filtered out from `anc_txids` later on. + let mut anc_txids: HashSet<_> = txs + .iter() + .flat_map(|tx| tx.input.iter().map(|txin| txin.previous_output.txid)) + .collect(); + while !anc_txids.is_empty() { + log::debug!("Syncing ancestor txids: {:?}", anc_txids); + let request = SyncRequest::from_chain_tip(local_chain.tip()) + .cache_graph_txs(&graph) + .chain_txids(anc_txids.clone()); + // We expect to have prev txouts for all unconfirmed ancestors in our graph so no need to fetch them here. + // Note we keep iterating through ancestors until we find one that is confirmed and only need to calculate + // fees for unconfirmed transactions. + let sync_result = self.sync_with_confirmation_height_anchor(request, false)?; + let _ = local_chain.apply_update(sync_result.chain_update); + if let Some(expected_tip) = &expected_tip { + if expected_tip != &local_chain.tip() { + return Err(Error::TipChanged( + expected_tip.block_id(), + local_chain.tip().block_id(), + )); + } + } + if local_chain.tip() != local_tip { + log::debug!("Chain tip changed while getting mempool entry. Restarting."); + return self.mempool_entries(txids, expected_tip); + } + let _ = graph.apply_update(sync_result.graph_update); + + // Add ancestors of any unconfirmed txs. + anc_txids = anc_txids + .iter() + .filter_map(|anc_txid| { + if let Some(ChainPosition::Unconfirmed(_)) = graph.get_chain_position( + &local_chain, + local_chain.tip().block_id(), + *anc_txid, + ) { + let anc_tx = graph.get_tx(*anc_txid).expect("we must have it"); + Some( + anc_tx + .input + .clone() + .iter() + .map(|txin| txin.previous_output.txid) + .collect::>(), + ) + } else { + None + } + }) + .flatten() + .collect(); + } + let mut entries = Vec::new(); + for tx in txs { + // Now iterate over ancestors and descendants in the graph. + let base_fee = graph + .calculate_fee(&tx) + .expect("all required txs are in graph"); + let base_size = tx.vsize(); + // Ancestor & descendant fees include those of `txid`. + let mut desc_fees = base_fee; + let mut anc_fees = base_fee; + // Ancestor size includes that of `txid`. + let mut anc_size = base_size; + for desc_txid in graph.walk_descendants(tx.txid(), |_, desc_txid| Some(desc_txid)) { + log::debug!("Getting fee for desc txid '{}'.", desc_txid); + let desc_tx = graph + .get_tx(desc_txid) + .expect("all descendant txs are in graph"); + let fee = graph + .calculate_fee(&desc_tx) + .expect("all required txs are in graph"); + desc_fees += fee; + } + for anc_tx in graph.walk_ancestors(tx, |_, anc_tx| Some(anc_tx)) { + log::debug!("Getting fee and size for anc txid '{}'.", anc_tx.txid()); + if let Some(ChainPosition::Unconfirmed(_)) = graph.get_chain_position( + &local_chain, + local_chain.tip().block_id(), + anc_tx.txid(), + ) { + let fee = graph + .calculate_fee(&anc_tx) + .expect("all required txs are in graph"); + anc_fees += fee; + anc_size += anc_tx.vsize(); + } else { + log::debug!("Ancestor txid '{}' is not unconfirmed.", anc_tx.txid()); + continue; + } + } + let fees = MempoolEntryFees { + base: bitcoin::Amount::from_sat(base_fee), + ancestor: bitcoin::Amount::from_sat(anc_fees), + descendant: bitcoin::Amount::from_sat(desc_fees), + }; + let entry = MempoolEntry { + vsize: base_size.try_into().expect("tx size must fit into u64"), + fees, + ancestor_vsize: anc_size.try_into().expect("tx size must fit into u64"), + }; + entries.push(entry) + } + + // It's possible that the chain tip has now changed, but it hadn't done as of the last sync, + // so go ahead and return the results. + Ok(entries) + } + + /// Get mempool entry for a single `txid`. + /// + /// Convenience method to call `mempool_entries` for a single `txid`, + /// returning `Option` instead of `Vec`. + pub fn mempool_entry(&self, txid: &bitcoin::Txid) -> Result, Error> { + // We just require the chain tip to stay the same while running `mempool_entries` so + // don't need to pass in an expected tip. + self.mempool_entries(HashSet::from([*txid]), None) + .map(|entries| entries.first().cloned()) + } + + /// Get mempool spenders of the given outpoints. + /// + /// Will restart if chain tip changes before completion. + pub fn mempool_spenders( + &self, + outpoints: &[bitcoin::OutPoint], + ) -> Result, Error> { + log::debug!("Getting mempool spenders for outpoints: {:?}.", outpoints); + let mut local_chain = LocalChain::from_genesis_hash(self.genesis_block()?.hash).0; + let chain_tip = self.chain_tip()?; + if chain_tip.height > 0 { + let _ = local_chain + .insert_block(block_id_from_tip(chain_tip)) + .expect("only contains genesis block"); + } + let request = + SyncRequest::from_chain_tip(local_chain.tip()).chain_outpoints(outpoints.to_vec()); + // We don't need to fetch prev txouts as we just want the outspends. + let sync_result = self.sync_with_confirmation_height_anchor(request, false)?; + let _ = local_chain.apply_update(sync_result.chain_update); + // Store tip at which first sync was completed. This will be our reference tip. + let local_tip = local_chain.tip(); + let graph = sync_result.graph_update; + let txids: HashSet<_> = outpoints + .iter() + .flat_map(|op| graph.outspends(*op)) + .copied() + .collect(); + let entries = match self.mempool_entries(txids, Some(local_tip)) { + Ok(entries) => entries, + Err(Error::TipChanged(expected, actual)) => { + log::debug!( + "Chain tip changed from {:?} to {:?} while \ + getting mempool spenders. Restarting.", + expected, + actual + ); + return self.mempool_spenders(outpoints); + } + Err(e) => { + return Err(e); + } + }; + Ok(entries) + } +} diff --git a/src/bitcoin/electrum/mod.rs b/src/bitcoin/electrum/mod.rs new file mode 100644 index 000000000..8b9745c76 --- /dev/null +++ b/src/bitcoin/electrum/mod.rs @@ -0,0 +1,250 @@ +use std::collections::HashMap; + +use bdk_electrum::bdk_chain::{ + bitcoin::{self, bip32::ChildNumber, BlockHash, OutPoint}, + local_chain::LocalChain, + spk_client::{FullScanRequest, SyncRequest}, + ChainPosition, +}; + +pub mod client; +mod utils; +pub mod wallet; +use crate::bitcoin::{Block, BlockChainTip, Coin}; + +/// An error in the Electrum interface. +#[derive(Debug)] +pub enum ElectrumError { + Client(client::Error), + GenesisHashMismatch( + BlockHash, /*expected hash*/ + BlockHash, /*server hash*/ + BlockHash, /*wallet hash*/ + ), +} + +impl std::fmt::Display for ElectrumError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + ElectrumError::Client(e) => write!(f, "Electrum client error: '{}'.", e), + ElectrumError::GenesisHashMismatch(expected, server, wallet) => { + write!( + f, + "Genesis hash mismatch. The genesis hash is expected to be '{}'. \ + The server has hash '{}' and the wallet has hash '{}'.", + expected, server, wallet, + ) + } + } + } +} + +/// Interface for Electrum backend. +pub struct Electrum { + client: client::Client, + bdk_wallet: wallet::BdkWallet, + /// Used for setting the `last_seen` of unconfirmed transactions in a strictly + /// increasing manner. + sync_count: u64, + /// Set to `true` to force a full scan from the genesis block regardless of + /// the wallet's local chain height. + full_scan: bool, +} + +impl Electrum { + pub fn new( + client: client::Client, + bdk_wallet: wallet::BdkWallet, + ) -> Result { + Ok(Self { + client, + bdk_wallet, + sync_count: 0, + full_scan: false, // by default, only perform full scan if wallet's local chain has height 0 + }) + } + + pub fn sanity_checks(&self, expected_hash: &bitcoin::BlockHash) -> Result<(), ElectrumError> { + let server_hash = self + .client + .genesis_block() + .map_err(ElectrumError::Client)? + .hash; + let wallet_hash = self.bdk_wallet.local_chain().genesis_hash(); + if server_hash != *expected_hash || wallet_hash != *expected_hash { + return Err(ElectrumError::GenesisHashMismatch( + *expected_hash, + server_hash, + wallet_hash, + )); + } + Ok(()) + } + + pub fn client(&self) -> &client::Client { + &self.client + } + + fn local_chain(&self) -> &LocalChain { + self.bdk_wallet.local_chain() + } + + /// Get all coins stored in the wallet, taking into consideration only those unconfirmed + /// transactions that were seen in the last wallet sync. + pub fn wallet_coins(&self, outpoints: Option<&[OutPoint]>) -> HashMap { + self.bdk_wallet.coins(outpoints, Some(self.sync_count)) + } + + /// Get the tip of the wallet's local chain. + pub fn wallet_tip(&self) -> BlockChainTip { + utils::tip_from_block_id(self.local_chain().tip().block_id()) + } + + /// Whether `tip` exists in the wallet's `local_chain`. + /// + /// Returns `None` if no block at that height exists in `local_chain`. + pub fn is_in_wallet_chain(&self, tip: BlockChainTip) -> Option { + self.bdk_wallet.is_in_chain(tip) + } + + /// Whether we'll perform a full scan at the next poll. + pub fn is_rescanning(&self) -> bool { + self.full_scan || self.local_chain().tip().height() == 0 + } + + /// Make the poller perform a full scan on the next iteration. + pub fn trigger_rescan(&mut self) { + self.full_scan = true; + } + + /// Sync the wallet with the Electrum server. If there was any reorg since the last poll, this + /// returns the first common ancestor between the previous and the new chain. + pub fn sync_wallet( + &mut self, + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Result, ElectrumError> { + self.bdk_wallet.reveal_spks(receive_index, change_index); + let local_chain_tip = self.local_chain().tip(); + log::debug!( + "local chain tip height before sync with electrum: {}", + local_chain_tip.block_id().height + ); + + // We'll only need to calculate fees of mempool transactions and this will be done separately from our graph + // so we don't need to fetch prev txouts. In any case, we'll already have these for our own transactions. + const FETCH_PREV_TXOUTS: bool = false; + const STOP_GAP: usize = 50; + + let (chain_update, mut graph_update, keychain_update) = if !self.is_rescanning() { + log::info!("Performing sync."); + let mut request = SyncRequest::from_chain_tip(local_chain_tip.clone()) + .cache_graph_txs(self.bdk_wallet.graph()); + + let all_spks: Vec<_> = self + .bdk_wallet + .index() + .inner() // we include lookahead SPKs + .all_spks() + .iter() + .map(|(_, script)| script.clone()) + .collect(); + request = request.chain_spks(all_spks); + log::debug!("num SPKs for sync: {}", request.spks.len()); + + let sync_result = self + .client + .sync_with_confirmation_time_height_anchor(request, FETCH_PREV_TXOUTS) + .map_err(ElectrumError::Client)?; + log::info!("Sync complete."); + (sync_result.chain_update, sync_result.graph_update, None) + } else { + log::info!("Performing full scan."); + // Either local_chain has height 0 or we want to trigger a full scan. + // In both cases, the scan should be from the genesis block. + let genesis_block = local_chain_tip.get(0).expect("must contain genesis block"); + let mut request = FullScanRequest::from_chain_tip(genesis_block) + .cache_graph_txs(self.bdk_wallet.graph()); + + for (k, spks) in self.bdk_wallet.index().all_unbounded_spk_iters() { + request = request.set_spks_for_keychain(k, spks); + } + let scan_result = self + .client + .full_scan_with_confirmation_time_height_anchor( + request, + STOP_GAP, + FETCH_PREV_TXOUTS, + ) + .map_err(ElectrumError::Client)?; + // A full scan only makes sense to do once, in most cases. Don't do it again unless + // explicitly asked to by a user. + self.full_scan = false; + log::info!("Full scan complete."); + ( + scan_result.chain_update, + scan_result.graph_update, + Some(scan_result.last_active_indices), + ) + }; + log::debug!( + "chain update height after sync with electrum: {}", + chain_update.height() + ); + + // Increment the sync count and apply changes. + self.sync_count = self.sync_count.checked_add(1).expect("must fit"); + if let Some(keychain_update) = keychain_update { + self.bdk_wallet.apply_keychain_update(keychain_update); + } + let changeset = self.bdk_wallet.apply_connected_chain_update(chain_update); + + let mut changes_iter = changeset.into_iter(); + let reorg_common_ancestor = loop { + match changes_iter.next() { + Some((height, Some(_))) => { + // `BlockHash` being `Some(_)` means a checkpoint at this height was added to the chain. + // Since we iterate in ascending height order, we'll see the lowest block height first. + // If the lowest height it adds is higher than our height before syncing, we're good. + // Else if it's adding a block at height before syncing or lower, it's a reorg. + break if height > local_chain_tip.height() { + None + } else { + log::info!("Block chain reorganization detected."); + Some(self.bdk_wallet.find_block_at_or_before_height(height)) + }; + } + Some((_, None)) => continue, + None => break None, + } + }; + + // Unconfirmed transactions have their last seen as 0, so we override to the `sync_count` + // so that conflicts can be properly handled. We use `sync_count` instead of current time + // in seconds to ensure strictly increasing values between poller iterations. + for tx in &graph_update.initial_changeset().txs { + let txid = tx.txid(); + if let Some(ChainPosition::Unconfirmed(_)) = graph_update.get_chain_position( + self.local_chain(), + self.local_chain().tip().block_id(), + txid, + ) { + log::debug!( + "changing last seen for txid '{}' to {}", + txid, + self.sync_count + ); + let _ = graph_update.insert_seen_at(txid, self.sync_count); + } + } + self.bdk_wallet.apply_graph_update(graph_update); + Ok(reorg_common_ancestor) + } + + pub fn wallet_transaction( + &self, + txid: &bitcoin::Txid, + ) -> Option<(bitcoin::Transaction, Option)> { + self.bdk_wallet.get_transaction(txid) + } +} diff --git a/src/bitcoin/electrum/utils.rs b/src/bitcoin/electrum/utils.rs new file mode 100644 index 000000000..2da51983b --- /dev/null +++ b/src/bitcoin/electrum/utils.rs @@ -0,0 +1,55 @@ +use std::convert::TryInto; + +use bdk_electrum::bdk_chain::{bitcoin, BlockId, ConfirmationTimeHeightAnchor}; + +use crate::bitcoin::{BlockChainTip, BlockInfo}; + +pub fn height_u32_from_i32(height: i32) -> u32 { + height.try_into().expect("height must fit into u32") +} + +pub fn height_i32_from_u32(height: u32) -> i32 { + height.try_into().expect("height must fit into i32") +} + +pub fn height_i32_from_usize(height: usize) -> i32 { + height.try_into().expect("height must fit into i32") +} + +pub fn height_usize_from_i32(height: i32) -> usize { + height.try_into().expect("height must fit into usize") +} + +pub fn block_id_from_tip(tip: BlockChainTip) -> BlockId { + BlockId { + height: height_u32_from_i32(tip.height), + hash: tip.hash, + } +} + +pub fn tip_from_block_id(id: BlockId) -> BlockChainTip { + BlockChainTip { + height: height_i32_from_u32(id.height), + hash: id.hash, + } +} + +pub fn block_info_from_anchor(anchor: ConfirmationTimeHeightAnchor) -> BlockInfo { + BlockInfo { + height: height_i32_from_u32(anchor.confirmation_height), + time: anchor + .confirmation_time + .try_into() + .expect("u32 by consensus"), + } +} + +/// Get the transaction's outpoints. +pub fn outpoints_from_tx(tx: &bitcoin::Transaction) -> Vec { + let txid = tx.txid(); + (0..tx.output.len()) + .map(|i| { + bitcoin::OutPoint::new(txid, i.try_into().expect("num tx outputs must fit in u32")) + }) + .collect::>() +} diff --git a/src/bitcoin/electrum/wallet.rs b/src/bitcoin/electrum/wallet.rs new file mode 100644 index 000000000..2e467f214 --- /dev/null +++ b/src/bitcoin/electrum/wallet.rs @@ -0,0 +1,328 @@ +use std::{ + collections::{BTreeMap, HashMap}, + convert::TryInto, + sync::Arc, +}; + +use bdk_electrum::bdk_chain::{ + bitcoin::{self, bip32, BlockHash, OutPoint, ScriptBuf, TxOut}, + keychain::KeychainTxOutIndex, + local_chain::{ChangeSet as ChainChangeSet, CheckPoint, LocalChain}, + miniscript::{Descriptor, DescriptorPublicKey}, + tx_graph::{self, TxGraph}, + ChainOracle, ChainPosition, ConfirmationTimeHeightAnchor, IndexedTxGraph, +}; +use miniscript::bitcoin::bip32::ChildNumber; + +use super::utils::{ + block_id_from_tip, block_info_from_anchor, height_i32_from_u32, height_u32_from_i32, +}; +use crate::{ + bitcoin::{Block, BlockChainTip, Coin, COINBASE_MATURITY}, + descriptors::LianaDescriptor, +}; + +// TODO: Move and reuse `liana::database::sqlite::utils::LOOK_AHEAD_LIMIT`? +const LOOK_AHEAD_LIMIT: u32 = 200; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum KeychainType { + Receive, + Change, +} + +pub struct BdkWallet { + graph: IndexedTxGraph>, + local_chain: LocalChain, + // Store descriptors for use when getting SPKs. + receive_desc: Descriptor, + change_desc: Descriptor, +} + +impl BdkWallet { + /// Create a new BDK wallet and initialize with the given data that was + /// valid as of `tip`. + /// + /// If there is no `tip`, then any provided data will be ignored. + /// + /// `receive_index` and `change_index` are the last used derivation + /// indices for the receive and change descriptors, respectively. + pub fn new( + main_descriptor: &LianaDescriptor, + genesis_hash: BlockHash, + tip: Option, + coins: &[Coin], + txs: &[bitcoin::Transaction], + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Self { + let local_chain = LocalChain::from_genesis_hash(genesis_hash).0; + let receive_desc = main_descriptor + .receive_descriptor() + .as_descriptor_public_key(); + let change_desc = main_descriptor + .change_descriptor() + .as_descriptor_public_key(); + + let mut bdk_wallet = BdkWallet { + graph: { + let mut indexer = KeychainTxOutIndex::::new(LOOK_AHEAD_LIMIT); + let _ = indexer.insert_descriptor(KeychainType::Receive, receive_desc.clone()); + let _ = indexer.insert_descriptor(KeychainType::Change, change_desc.clone()); + IndexedTxGraph::new(indexer) + }, + local_chain, + receive_desc: receive_desc.clone(), + change_desc: change_desc.clone(), + }; + if let Some(tip) = tip { + // This will be our anchor for any confirmed transactions. + let anchor_block = block_id_from_tip(tip); + if tip.height > 0 { + log::debug!("inserting block into local chain: {:?}", anchor_block); + let _ = bdk_wallet + .local_chain + .insert_block(anchor_block) + .expect("local chain only contains genesis block"); + } + // Update the last used derivation index for both change and receive addresses. + log::debug!( + "revealing SPKs up to receive index {receive_index} and change index {change_index}" + ); + bdk_wallet.reveal_spks(receive_index, change_index); + + // Update the existing coins and transactions information using a TxGraph changeset. + log::debug!("Number of coins to load: {}.", coins.len()); + log::debug!("Number of txs to load: {}.", txs.len()); + let mut graph_cs = tx_graph::ChangeSet::default(); + for tx in txs { + graph_cs.txs.insert(Arc::new(tx.clone())); + } + for coin in coins { + // First of all insert the txout itself. + let script_pubkey = bdk_wallet.get_spk(coin.derivation_index, coin.is_change); + let txout = TxOut { + script_pubkey, + value: coin.amount, + }; + graph_cs.txouts.insert(coin.outpoint, txout); + // If the coin's deposit transaction is confirmed, tell BDK by inserting an anchor. + // Otherwise, we could insert a last seen timestamp but we don't have such data stored in + // the table. + if let Some(block) = coin.block_info { + graph_cs.anchors.insert(( + ConfirmationTimeHeightAnchor { + confirmation_height: height_u32_from_i32(block.height), + confirmation_time: block.time.into(), + anchor_block, + }, + coin.outpoint.txid, + )); + } + // If the coin's spending transaction is confirmed, do the same. + if let Some(block) = coin.spend_block { + let spend_txid = coin.spend_txid.expect("Must be present if confirmed."); + graph_cs.anchors.insert(( + ConfirmationTimeHeightAnchor { + confirmation_height: height_u32_from_i32(block.height), + confirmation_time: block.time.into(), + anchor_block, + }, + spend_txid, + )); + } + } + let mut graph = TxGraph::default(); + graph.apply_changeset(graph_cs); + let _ = bdk_wallet.graph.apply_update(graph); + } + bdk_wallet + } + + /// Get a reference to the local chain. + pub fn local_chain(&self) -> &LocalChain { + &self.local_chain + } + + /// Whether `tip` exists in `local_chain`. + /// + /// Returns `None` if no block at that height exists in `local_chain`. + pub fn is_in_chain(&self, tip: BlockChainTip) -> Option { + self.local_chain + .is_block_in_chain(block_id_from_tip(tip), self.local_chain().tip().block_id()) + .expect("function is infallible") + } + + /// Get a reference to the graph. + pub fn graph(&self) -> &TxGraph { + self.graph.graph() + } + + /// Get a reference to the transaction index. + pub fn index(&self) -> &KeychainTxOutIndex { + &self.graph.index + } + + /// Reveal SPKs based on derivation indices set in DB. + pub fn reveal_spks(&mut self, receive_index: ChildNumber, change_index: ChildNumber) { + let mut keychain_update = BTreeMap::new(); + keychain_update.insert(KeychainType::Receive, receive_index.into()); + keychain_update.insert(KeychainType::Change, change_index.into()); + self.apply_keychain_update(keychain_update) + } + + fn get_spk(&self, der_index: bip32::ChildNumber, is_change: bool) -> ScriptBuf { + // Try to get it from the BDK wallet cache first, failing that derive it from the appropriate + // descriptor. + let chain_kind = if is_change { + KeychainType::Change + } else { + KeychainType::Receive + }; + if let Some(spk) = self.graph.index.spk_at_index(chain_kind, der_index.into()) { + spk.to_owned() + } else { + let desc = if is_change { + &self.change_desc + } else { + &self.receive_desc + }; + desc.at_derivation_index(der_index.into()) + .expect("Not multipath and index isn't hardened.") + .script_pubkey() + } + } + + /// Get the coins currently stored by the `BdkWallet` optionally filtered by `outpoints`. + /// If `outpoints` is `None`, no filter will be applied. + /// If `outpoints` is an empty slice, no coins will be returned. + /// If `last_seen` is set, only those unconfirmed transactions with a matching last seen + /// will be considered. + pub fn coins( + &self, + outpoints: Option<&[bitcoin::OutPoint]>, + last_seen: Option, + ) -> HashMap { + // Get an iterator over all the wallet txos (not only the currently unspent ones) by using + // lower level methods. + let tx_graph = self.graph.graph(); + let txo_index = &self.graph.index; + let tip_id = self.local_chain.tip().block_id(); + let wallet_txos = + tx_graph.filter_chain_txouts(&self.local_chain, tip_id, txo_index.outpoints()); + let mut wallet_coins = HashMap::new(); + // Go through all the wallet txos and create a coin for each. + for ((k, i), full_txo) in wallet_txos { + let outpoint = full_txo.outpoint; + if outpoints.map(|ops| !ops.contains(&outpoint)) == Some(true) { + continue; + } + let amount = full_txo.txout.value; + let derivation_index = i.into(); + let is_change = matches!(k, KeychainType::Change); + let block_info = match full_txo.chain_position { + ChainPosition::Unconfirmed(ls) => { + if let Some(last_seen) = last_seen.filter(|last_seen| *last_seen != ls) { + log::debug!("Ignoring coin at {}, which was last seen at {} instead of {} as required.", outpoint, ls, last_seen); + continue; + } + None + } + ChainPosition::Confirmed(anchor) => Some(block_info_from_anchor(anchor)), + }; + + // Immature if from a coinbase transaction with less than a hundred confs. + let is_immature = full_txo.is_on_coinbase + && block_info + .and_then(|blk| { + let tip_height: i32 = height_i32_from_u32(tip_id.height); + tip_height + .checked_sub(blk.height) + .map(|confs| confs < COINBASE_MATURITY) + }) + .unwrap_or(true); + + // Get spend status of this coin. + let (mut spend_txid, mut spend_block) = (None, None); + if let Some((spend_pos, txid)) = full_txo.spent_by { + spend_txid = Some(txid); + match spend_pos { + ChainPosition::Unconfirmed(ls) => { + if let Some(last_seen) = last_seen.filter(|last_seen| *last_seen != ls) { + log::debug!( + "Ignoring spend txid {} for coin at {}, \ + which was last seen at {} instead of {} as required.", + txid, + outpoint, + ls, + last_seen + ); + spend_txid = None; + } + } + ChainPosition::Confirmed(anchor) => { + spend_block = Some(block_info_from_anchor(anchor)); + } + }; + } + let coin = crate::bitcoin::Coin { + outpoint, + amount, + derivation_index, + is_change, + is_immature, + block_info, + spend_txid, + spend_block, + }; + wallet_coins.insert(coin.outpoint, coin); + } + wallet_coins + } + + pub fn get_transaction( + &self, + txid: &bitcoin::Txid, + ) -> Option<(bitcoin::Transaction, Option)> { + self.graph.graph().get_tx_node(*txid).map(|tx_node| { + let block = tx_node.anchors.iter().next().map(|info| Block { + hash: info.anchor_block.hash, // not necessarily the confirmation block hash + height: height_i32_from_u32(info.confirmation_height), + time: info.confirmation_time.try_into().expect("u32 by consensus"), + }); + let tx = tx_node.tx.as_ref().clone(); + (tx, block) + }) + } + + /// Find the first block in the local chain whose height is less than or equal to this. + pub fn find_block_at_or_before_height(&self, height: u32) -> BlockChainTip { + for cp in self.local_chain.iter_checkpoints() { + if cp.height() <= height { + return BlockChainTip { + height: height_i32_from_u32(cp.height()), + hash: cp.hash(), + }; + } + } + unreachable!("There must be at least the genesis block.") + } + + /// Apply an update to the local chain. + /// Panics if update does not connect to the local chain. + pub fn apply_connected_chain_update(&mut self, chain_update: CheckPoint) -> ChainChangeSet { + self.local_chain + .apply_update(chain_update) + .expect("update must connect to local chain") + } + + /// Apply a graph update. + pub fn apply_graph_update(&mut self, graph_update: TxGraph) { + let _ = self.graph.apply_update(graph_update); + } + + /// Apply a keychain update. + pub fn apply_keychain_update(&mut self, keychain_update: BTreeMap) { + let _ = self.graph.index.reveal_to_target_multi(&keychain_update); + } +} diff --git a/src/bitcoin/mod.rs b/src/bitcoin/mod.rs index 9ddc8a98c..104d80d6f 100644 --- a/src/bitcoin/mod.rs +++ b/src/bitcoin/mod.rs @@ -3,6 +3,7 @@ //! Broadcast transactions, poll for new unspent coins, gather fee estimates. pub mod d; +pub mod electrum; pub mod poller; use crate::{ @@ -405,6 +406,193 @@ impl BitcoinInterface for d::BitcoinD { } } +impl BitcoinInterface for electrum::Electrum { + fn sync_wallet( + &mut self, + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Result, String> { + self.sync_wallet(receive_index, change_index) + .map_err(|e| e.to_string()) + } + + fn received_coins( + &self, + tip: &BlockChainTip, + _descs: &[descriptors::SinglePathLianaDesc], + ) -> Vec { + // Get those wallet coins that are either unconfirmed or have a confirmation height + // after tip. The poller will then discard any that had already been received. + self.wallet_coins(None) + .values() + .filter_map(|c| { + let height = c.block_info.map(|info| info.height); + if height.filter(|h| *h <= tip.height).is_some() { + None + } else { + Some(UTxO { + outpoint: c.outpoint, + block_height: height, + amount: c.amount, + address: UTxOAddress::DerivIndex(c.derivation_index, c.is_change), + is_immature: c.is_immature, + }) + } + }) + .collect() + } + + fn confirmed_coins( + &self, + outpoints: &[bitcoin::OutPoint], + ) -> (Vec<(bitcoin::OutPoint, i32, u32)>, Vec) { + let wallet_coins = &self.wallet_coins(Some(outpoints)); + let mut confirmed = Vec::new(); + let mut expired = Vec::new(); + for op in outpoints { + if let Some(w_c) = wallet_coins.get(op) { + if let Some(block) = w_c.block_info { + if w_c.is_immature { + log::debug!( + "Coin at '{}' comes from an immature coinbase transaction at \ + block height {}. Not marking it as confirmed for now.", + op, + block.height + ); + continue; + } + confirmed.push((w_c.outpoint, block.height, block.time)); + } + } else { + expired.push(*op); + } + } + (confirmed, expired) + } + + fn spending_coins( + &self, + outpoints: &[bitcoin::OutPoint], + ) -> Vec<(bitcoin::OutPoint, bitcoin::Txid)> { + let wallet_coins = &self.wallet_coins(Some(outpoints)); + outpoints + .iter() + .filter_map(|op| { + if let Some(w_c) = wallet_coins.get(op) { + w_c.spend_txid.map(|txid| (w_c.outpoint, txid)) + } else { + None + } + }) + .collect() + } + + fn spent_coins( + &self, + outpoints: &[(bitcoin::OutPoint, bitcoin::Txid)], + ) -> (Vec, Vec) { + let ops: Vec<_> = outpoints.iter().map(|(op, _)| op).copied().collect(); + let wallet_coins = &self.wallet_coins(Some(&ops)); + let mut spent = Vec::new(); + let mut expired_spending = Vec::new(); + + for (op, spend_txid) in outpoints { + if let Some(w_c) = wallet_coins.get(op) { + if w_c.spend_txid != Some(*spend_txid) { + expired_spending.push(*op); + } + if let Some(block) = w_c.spend_block { + spent.push((*op, *spend_txid, block.height, block.time)); + } + } + } + (spent, expired_spending) + } + + fn genesis_block_timestamp(&self) -> u32 { + self.client() + .genesis_block_timestamp() + .expect("Genesis block timestamp must always be there") + } + + fn genesis_block(&self) -> BlockChainTip { + self.client() + .genesis_block() + .expect("Genesis block must always be there") + } + + fn chain_tip(&self) -> BlockChainTip { + // We want the wallet's local chain tip after syncing. + self.wallet_tip() + } + + fn is_in_chain(&self, tip: &BlockChainTip) -> bool { + // Return `false` if no block at same height as `tip` + // is in wallet's local chain. + self.is_in_wallet_chain(*tip).unwrap_or_default() + } + + /// FIXME: make the Bitcoin backend interface higher level. See the comment in the poller next + /// to the `sync_wallet()` call. + fn common_ancestor(&self, _tip: &BlockChainTip) -> Option { + unreachable!("The common ancestor is returned in `sync_wallet()`. If no reorg was detected then, this method will never be called on an Electrum backend.") + } + + fn broadcast_tx(&self, tx: &bitcoin::Transaction) -> Result<(), String> { + match self.client().broadcast_tx(tx) { + Ok(_txid) => Ok(()), + Err(e) => Err(e.to_string()), + } + } + + fn wallet_transaction( + &self, + txid: &bitcoin::Txid, + ) -> Option<(bitcoin::Transaction, Option)> { + self.wallet_transaction(txid) + } + + fn mempool_entry(&self, txid: &bitcoin::Txid) -> Option { + self.client().mempool_entry(txid).ok()? + } + + fn mempool_spenders(&self, outpoints: &[bitcoin::OutPoint]) -> Vec { + self.client() + .mempool_spenders(outpoints) + .unwrap_or_default() + } + + fn sync_progress(&self) -> SyncProgress { + // Always return 100% for now since the API is bitcoind-specific to mean "blocks/headers". + // But in the future it would be nice to inform the user about the progress of the sync + // if it takes a few dozen seconds. + let blocks = self.chain_tip().height as u64; + SyncProgress::new(1.0, blocks, blocks) + } + + fn start_rescan( + &mut self, + _desc: &descriptors::LianaDescriptor, + _timestamp: u32, + ) -> Result<(), String> { + self.trigger_rescan(); + Ok(()) + } + + fn rescan_progress(&self) -> Option { + // Until we sync we're at 0%. After the sync, we're at 100%. + self.is_rescanning().then_some(0.0) + } + + fn block_before_date(&self, _timestamp: u32) -> Option { + Some(self.genesis_block()) + } + + fn tip_time(&self) -> Option { + self.client().tip_time().ok() + } +} + // FIXME: do we need to repeat the entire trait implemenation? Isn't there a nicer way? impl BitcoinInterface for sync::Arc> { fn genesis_block_timestamp(&self) -> u32 { @@ -528,3 +716,21 @@ pub enum UTxOAddress { /// Derivation index and whether it is from the change descriptor. DerivIndex(ChildNumber, bool), } + +#[derive(Debug, Clone, Copy)] +pub struct BlockInfo { + pub height: i32, + pub time: u32, +} + +#[derive(Debug, Clone, Copy)] +pub struct Coin { + pub outpoint: bitcoin::OutPoint, + pub amount: bitcoin::Amount, + pub derivation_index: ChildNumber, + pub is_change: bool, + pub is_immature: bool, + pub block_info: Option, + pub spend_txid: Option, + pub spend_block: Option, +} diff --git a/src/config.rs b/src/config.rs index 6312b9e46..d82cf32bd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -91,6 +91,9 @@ pub enum BitcoinBackend { /// Settings specific to bitcoind as the Bitcoin interface. #[serde(rename = "bitcoind_config")] Bitcoind(BitcoindConfig), + /// Settings specific to Electrum as the Bitcoin interface. + #[serde(rename = "electrum_config")] + Electrum(ElectrumConfig), } /// RPC authentication options. @@ -123,6 +126,15 @@ pub struct BitcoindConfig { pub addr: SocketAddr, } +/// Everything we need to know for talking to Electrum serenely. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ElectrumConfig { + /// The URL the Electrum's RPC is listening on. + /// Include "ssl://" for SSL. otherwise TCP will be assumed. + /// Can optionally prefix with "tcp://". + pub addr: String, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct BitcoinConfig { /// The network we are operating on, one of "bitcoin", "testnet", "regtest", "signet" diff --git a/src/lib.rs b/src/lib.rs index fa30706ac..67b489fe8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,9 +14,13 @@ pub mod spend; mod testutils; pub use bip39; +use bitcoin::electrum; pub use miniscript; -pub use crate::bitcoin::d::{BitcoinD, BitcoindError, WalletError}; +pub use crate::bitcoin::{ + d::{BitcoinD, BitcoindError, WalletError}, + electrum::{Electrum, ElectrumError}, +}; #[cfg(feature = "daemon")] use crate::jsonrpc::server::{rpcserver_loop, rpcserver_setup}; use crate::{ @@ -34,7 +38,7 @@ use std::{ thread, }; -use miniscript::bitcoin::secp256k1; +use miniscript::bitcoin::{constants::ChainHash, hashes::Hash, secp256k1, BlockHash}; #[cfg(not(test))] use std::panic; @@ -92,10 +96,12 @@ pub enum StartupError { DefaultDataDirNotFound, DatadirCreation(path::PathBuf, io::Error), MissingBitcoindConfig, + MissingElectrumConfig, MissingBitcoinBackendConfig, DbMigrateBitcoinTxs(&'static str), Database(SqliteDbError), Bitcoind(BitcoindError), + Electrum(ElectrumError), #[cfg(unix)] Daemonization(&'static str), #[cfg(windows)] @@ -118,6 +124,10 @@ impl fmt::Display for StartupError { f, "Our Bitcoin interface is bitcoind but we have no 'bitcoind_config' entry in the configuration." ), + Self::MissingElectrumConfig => write!( + f, + "Our Bitcoin interface is Electrum but we have no 'electrum_config' entry in the configuration." + ), Self::MissingBitcoinBackendConfig => write!( f, "No Bitcoin backend entry in the configuration." @@ -128,6 +138,7 @@ impl fmt::Display for StartupError { ), Self::Database(e) => write!(f, "Error initializing database: '{}'.", e), Self::Bitcoind(e) => write!(f, "Error setting up bitcoind interface: '{}'.", e), + Self::Electrum(e) => write!(f, "Error setting up Electrum interface: '{}'.", e), #[cfg(unix)] Self::Daemonization(e) => write!(f, "Error when daemonizing: '{}'.", e), #[cfg(windows)] @@ -265,10 +276,10 @@ fn setup_bitcoind( #[cfg(target_os = "windows")] let wo_path_str = wo_path_str.replace("\\\\?\\", "").replace("\\\\?", ""); - let config::BitcoinBackend::Bitcoind(bitcoind_config) = config - .bitcoin_backend - .as_ref() - .ok_or(StartupError::MissingBitcoindConfig)?; + let bitcoind_config = match config.bitcoin_backend.as_ref() { + Some(config::BitcoinBackend::Bitcoind(bitcoind_config)) => bitcoind_config, + _ => Err(StartupError::MissingBitcoindConfig)?, + }; let bitcoind = BitcoinD::new(bitcoind_config, wo_path_str)?; bitcoind.node_sanity_checks( config.bitcoin_config.network, @@ -292,6 +303,70 @@ fn setup_bitcoind( Ok(bitcoind) } +// Create an Electrum interface from a client and BDK-based wallet, and do some sanity checks. +// If all went well, returns the interface to Electrum. +fn setup_electrum( + config: &Config, + db: sync::Arc>, +) -> Result { + let electrum_config = match config.bitcoin_backend.as_ref() { + Some(config::BitcoinBackend::Electrum(electrum_config)) => electrum_config, + _ => Err(StartupError::MissingElectrumConfig)?, + }; + // First create the client to communicate with the Electrum server. + let client = electrum::client::Client::new(electrum_config) + .map_err(|e| StartupError::Electrum(ElectrumError::Client(e)))?; + // Then create the BDK-based wallet and populate it with DB data. + let mut db_conn = db.connection(); + let tip = db_conn.chain_tip(); + let coins: Vec<_> = db_conn + .coins(&[], &[]) + .into_values() + .map(|c| crate::bitcoin::Coin { + outpoint: c.outpoint, + amount: c.amount, + derivation_index: c.derivation_index, + is_change: c.is_change, + is_immature: c.is_immature, + block_info: c.block_info.map(|info| crate::bitcoin::BlockInfo { + height: info.height, + time: info.time, + }), + spend_txid: c.spend_txid, + spend_block: c.spend_block.map(|info| crate::bitcoin::BlockInfo { + height: info.height, + time: info.time, + }), + }) + .collect(); + let txids = db_conn.list_saved_txids(); + // This will only return those txs referenced by our coins, which may not be all of `txids`. + let txs: Vec<_> = db_conn + .list_wallet_transactions(&txids) + .into_iter() + .map(|(tx, _, _)| tx) + .collect(); + let (receive_index, change_index) = (db_conn.receive_index(), db_conn.change_index()); + let genesis_hash = { + let chain_hash = ChainHash::using_genesis_block(config.bitcoin_config.network); + BlockHash::from_byte_array(*chain_hash.as_bytes()) + }; + let bdk_wallet = electrum::wallet::BdkWallet::new( + &config.main_descriptor, + genesis_hash, + tip, + &coins, + &txs, + receive_index, + change_index, + ); + let electrum = Electrum::new(client, bdk_wallet).map_err(StartupError::Electrum)?; + electrum + .sanity_checks(&genesis_hash) + .map_err(StartupError::Electrum)?; + Ok(electrum) +} + #[derive(Clone)] pub struct DaemonControl { config: Config, @@ -407,6 +482,9 @@ impl DaemonHandle { sync::Mutex::from(bitcoind.expect("bitcoind must have been set already")), ) as sync::Arc>, + (None, Some(config::BitcoinBackend::Electrum(..))) => { + sync::Arc::from(sync::Mutex::from(setup_electrum(&config, db.clone())?)) + } (None, None) => Err(StartupError::MissingBitcoinBackendConfig)?, }; From 1b04b294910d1cb65885f17dd266029771b476cf Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Wed, 21 Aug 2024 18:44:13 +0100 Subject: [PATCH 10/17] func test: fix min rbf feerate Here, the min RBF feerate is 1 more than that of the transaction to be replaced. The feerate of the transaction to be replaced may vary slightly depending on the signature size and is calculated during the test. As such, the min feerate should be set according to this calculated value. --- tests/test_rpc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_rpc.py b/tests/test_rpc.py index 675f8001f..cd697aae1 100644 --- a/tests/test_rpc.py +++ b/tests/test_rpc.py @@ -1145,7 +1145,8 @@ def test_rbfpsbt_bump_fee(lianad, bitcoind): # feerate to set the min feerate, instead of 1 sat/vb of first # transaction: with pytest.raises( - RpcError, match=f"Feerate {int(rbf_1_feerate)} too low for minimum feerate 10." + RpcError, + match=f"Feerate {int(rbf_1_feerate)} too low for minimum feerate {int(rbf_1_feerate) + 1}.", ): lianad.rpc.rbfpsbt(first_txid, False, int(rbf_1_feerate)) # Using 1 more for feerate works. From a85d4887e9efeaada6a1a79f4a35a32cf8444569 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Thu, 22 Aug 2024 12:02:08 +0100 Subject: [PATCH 11/17] func test: allow for different bitcoin backends --- tests/fixtures.py | 35 ++++++++++++++++++++------------ tests/test_framework/bitcoind.py | 18 ++++++++++++++-- tests/test_framework/lianad.py | 8 ++------ tests/test_framework/utils.py | 21 +++++++++++++++++++ 4 files changed, 61 insertions(+), 21 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 68b6e9ee1..fd42ab0c2 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -5,7 +5,12 @@ from test_framework.bitcoind import Bitcoind from test_framework.lianad import Lianad from test_framework.signer import SingleSigner, MultiSigner -from test_framework.utils import EXECUTOR_WORKERS, USE_TAPROOT +from test_framework.utils import ( + BITCOIN_BACKEND_TYPE, + EXECUTOR_WORKERS, + USE_TAPROOT, + BitcoinBackendType, +) import hashlib import os @@ -115,6 +120,16 @@ def bitcoind(directory): bitcoind.cleanup() +@pytest.fixture +def bitcoin_backend(directory, bitcoind): + + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + yield bitcoind + bitcoind.cleanup() + else: + raise NotImplementedError + + def xpub_fingerprint(hd): return _pubkey_to_fingerprint(hd.pubkey).hex() @@ -127,10 +142,9 @@ def single_key_desc(prim_fg, prim_xpub, reco_fg, reco_xpub, csv_value, is_taproo @pytest.fixture -def lianad(bitcoind, directory): +def lianad(bitcoin_backend, directory): datadir = os.path.join(directory, "lianad") os.makedirs(datadir, exist_ok=True) - bitcoind_cookie = os.path.join(bitcoind.bitcoin_dir, "regtest", ".cookie") signer = SingleSigner(is_taproot=USE_TAPROOT) (prim_fingerprint, primary_xpub), (reco_fingerprint, recovery_xpub) = ( @@ -155,8 +169,7 @@ def lianad(bitcoind, directory): datadir, signer, main_desc, - bitcoind.rpcport, - bitcoind_cookie, + bitcoin_backend, ) try: @@ -208,10 +221,9 @@ def multisig_desc(multi_signer, csv_value, is_taproot): @pytest.fixture -def lianad_multisig(bitcoind, directory): +def lianad_multisig(bitcoin_backend, directory): datadir = os.path.join(directory, "lianad") os.makedirs(datadir, exist_ok=True) - bitcoind_cookie = os.path.join(bitcoind.bitcoin_dir, "regtest", ".cookie") # A 3-of-4 that degrades into a 2-of-5 after 10 blocks csv_value = 10 @@ -224,8 +236,7 @@ def lianad_multisig(bitcoind, directory): datadir, signer, main_desc, - bitcoind.rpcport, - bitcoind_cookie, + bitcoin_backend, ) try: @@ -261,10 +272,9 @@ def multipath_desc(multi_signer, csv_values, is_taproot): @pytest.fixture -def lianad_multipath(bitcoind, directory): +def lianad_multipath(bitcoin_backend, directory): datadir = os.path.join(directory, "lianad") os.makedirs(datadir, exist_ok=True) - bitcoind_cookie = os.path.join(bitcoind.bitcoin_dir, "regtest", ".cookie") # A 3-of-4 that degrades into a 3-of-5 after 10 blocks and into a 1-of-10 after 20 blocks. csv_values = [10, 20] @@ -279,8 +289,7 @@ def lianad_multipath(bitcoind, directory): datadir, signer, main_desc, - bitcoind.rpcport, - bitcoind_cookie, + bitcoin_backend, ) try: diff --git a/tests/test_framework/bitcoind.py b/tests/test_framework/bitcoind.py index ab8e69ac3..95ae04578 100644 --- a/tests/test_framework/bitcoind.py +++ b/tests/test_framework/bitcoind.py @@ -7,7 +7,14 @@ from decimal import Decimal from ephemeral_port_reserve import reserve from test_framework.authproxy import AuthServiceProxy -from test_framework.utils import TailableProc, wait_for, TIMEOUT, BITCOIND_PATH, COIN +from test_framework.utils import ( + BitcoinBackend, + TailableProc, + wait_for, + TIMEOUT, + BITCOIND_PATH, + COIN, +) class BitcoindRpcInterface: @@ -35,7 +42,7 @@ def f(*args): return f -class Bitcoind(TailableProc): +class Bitcoind(BitcoinBackend): def __init__(self, bitcoin_dir, rpcport=None): TailableProc.__init__(self, bitcoin_dir, verbose=False) @@ -275,3 +282,10 @@ def cleanup(self): except Exception: self.proc.kill() self.proc.wait() + + def append_to_lianad_conf(self, conf_file): + cookie_path = os.path.join(self.bitcoin_dir, "regtest", ".cookie") + with open(conf_file, "a") as f: + f.write("[bitcoind_config]\n") + f.write(f"cookie_path = '{cookie_path}'\n") + f.write(f"addr = '127.0.0.1:{self.rpcport}'\n") diff --git a/tests/test_framework/lianad.py b/tests/test_framework/lianad.py index dfd81cd9d..d9b4afdb6 100644 --- a/tests/test_framework/lianad.py +++ b/tests/test_framework/lianad.py @@ -28,8 +28,7 @@ def __init__( datadir, signer, multi_desc, - bitcoind_rpc_port, - bitcoind_cookie_path, + bitcoin_backend, ): TailableProc.__init__(self, datadir, verbose=VERBOSE) @@ -55,10 +54,7 @@ def __init__( f.write("[bitcoin_config]\n") f.write('network = "regtest"\n') f.write("poll_interval_secs = 1\n") - - f.write("[bitcoind_config]\n") - f.write(f"cookie_path = '{bitcoind_cookie_path}'\n") - f.write(f"addr = '127.0.0.1:{bitcoind_rpc_port}'\n") + bitcoin_backend.append_to_lianad_conf(self.conf_file) def finalize_psbt(self, psbt): """Create a valid witness for all inputs in the PSBT. diff --git a/tests/test_framework/utils.py b/tests/test_framework/utils.py index 7f961038b..d92f862f9 100644 --- a/tests/test_framework/utils.py +++ b/tests/test_framework/utils.py @@ -1,3 +1,5 @@ +import abc +import enum import itertools import json import logging @@ -20,6 +22,16 @@ os.path.dirname(__file__), "..", "..", "target/debug/lianad" ) LIANAD_PATH = os.getenv("LIANAD_PATH", DEFAULT_MS_PATH) + + +class BitcoinBackendType(str, enum.Enum): + Bitcoind = "bitcoind" + + +DEFAULT_BITCOIN_BACKEND_TYPE = "bitcoind" +BITCOIN_BACKEND_TYPE = BitcoinBackendType( + os.getenv("BITCOIN_BACKEND_TYPE", DEFAULT_BITCOIN_BACKEND_TYPE) +) DEFAULT_BITCOIND_PATH = "bitcoind" BITCOIND_PATH = os.getenv("BITCOIND_PATH", DEFAULT_BITCOIND_PATH) OLD_LIANAD_PATH = os.getenv("OLD_LIANAD_PATH", None) @@ -421,3 +433,12 @@ def wait_for_log(self, regex, timeout=TIMEOUT): Convenience wrapper for the common case of only seeking a single entry. """ return self.wait_for_logs([regex], timeout) + + +class BitcoinBackend(abc.ABC, TailableProc): + """All Bitcoin backends should derive from this class.""" + + @abc.abstractmethod + def append_to_lianad_conf(self, conf_file): + """Append backend config values to lianad config file.""" + ... From 371e31e3f35ac5054fe2bd7dd04bcdd8790e8988 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:00:08 +0100 Subject: [PATCH 12/17] func test: allow to run using electrs backend --- tests/fixtures.py | 11 +++++ tests/test_chain.py | 12 +++-- tests/test_framework/electrs.py | 79 +++++++++++++++++++++++++++++++++ tests/test_framework/lianad.py | 8 +++- tests/test_framework/utils.py | 3 ++ tests/test_misc.py | 10 +++++ tests/test_rpc.py | 6 ++- 7 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 tests/test_framework/electrs.py diff --git a/tests/fixtures.py b/tests/fixtures.py index fd42ab0c2..4c362c835 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -3,6 +3,7 @@ from bip380.descriptors import Descriptor from concurrent import futures from test_framework.bitcoind import Bitcoind +from test_framework.electrs import Electrs from test_framework.lianad import Lianad from test_framework.signer import SingleSigner, MultiSigner from test_framework.utils import ( @@ -126,6 +127,16 @@ def bitcoin_backend(directory, bitcoind): if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: yield bitcoind bitcoind.cleanup() + elif BITCOIN_BACKEND_TYPE is BitcoinBackendType.Electrs: + electrs = Electrs( + electrs_dir=os.path.join(directory, "electrs"), + bitcoind_dir=bitcoind.bitcoin_dir, + bitcoind_rpcport=bitcoind.rpcport, + bitcoind_p2pport=bitcoind.p2pport, + ) + electrs.startup() + yield electrs + electrs.cleanup() else: raise NotImplementedError diff --git a/tests/test_chain.py b/tests/test_chain.py index 26487c999..b4bf5960c 100644 --- a/tests/test_chain.py +++ b/tests/test_chain.py @@ -238,7 +238,8 @@ def reorg_shift(height, txs): outpoints_before = set(c["outpoint"] for c in coins_before) bitcoind.generate_block(1) lianad.restart_fresh(bitcoind) - assert len(list_coins()) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(list_coins()) == 0 # We can be stopped while we are rescanning lianad.rpc.startrescan(initial_tip["time"]) @@ -252,7 +253,8 @@ def reorg_shift(height, txs): bitcoind.generate_block(1) lianad.restart_fresh(bitcoind) wait_for(lambda: lianad.rpc.getinfo()["rescan_progress"] is None) - assert len(list_coins()) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(list_coins()) == 0 # There can be a reorg when we start rescanning reorg_shift(initial_tip["height"], txs) @@ -271,7 +273,8 @@ def reorg_shift(height, txs): lianad.restart_fresh(bitcoind) wait_synced() wait_for(lambda: lianad.rpc.getinfo()["rescan_progress"] is None) - assert len(list_coins()) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(list_coins()) == 0 # We can be rescanning when a reorg happens lianad.rpc.startrescan(initial_tip["time"]) @@ -350,7 +353,8 @@ def test_rescan_and_recovery(lianad, bitcoind): # Clear lianad state lianad.restart_fresh(bitcoind) - assert len(lianad.rpc.listcoins()["coins"]) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(lianad.rpc.listcoins()["coins"]) == 0 # Start rescan lianad.rpc.startrescan(initial_tip["time"]) diff --git a/tests/test_framework/electrs.py b/tests/test_framework/electrs.py new file mode 100644 index 000000000..ee8c6a518 --- /dev/null +++ b/tests/test_framework/electrs.py @@ -0,0 +1,79 @@ +import logging +import os + +from ephemeral_port_reserve import reserve +from test_framework.utils import BitcoinBackend, TailableProc, ELECTRS_PATH + + +class Electrs(BitcoinBackend): + def __init__( + self, + bitcoind_dir, + bitcoind_rpcport, + bitcoind_p2pport, + electrs_dir, + rpcport=None, + ): + TailableProc.__init__(self, electrs_dir, verbose=False) + + if rpcport is None: + rpcport = reserve() + + # Prometheus metrics can't be deactivated in Electrs. Configure the port so it doesn't + # conflict with other instances when running tests in parallel. + monitoring_port = reserve() + + self.electrs_dir = electrs_dir + self.rpcport = rpcport + + regtestdir = os.path.join(electrs_dir, "regtest") + if not os.path.exists(regtestdir): + os.makedirs(regtestdir) + + self.cmd_line = [ + ELECTRS_PATH, + "--conf", + "{}/electrs.toml".format(regtestdir), + ] + electrs_conf = { + "daemon_dir": bitcoind_dir, + "cookie_file": os.path.join(bitcoind_dir, "regtest", ".cookie"), + "daemon_rpc_addr": f"127.0.0.1:{bitcoind_rpcport}", + "daemon_p2p_addr": f"127.0.0.1:{bitcoind_p2pport}", + "db_dir": electrs_dir, + "network": "regtest", + "electrum_rpc_addr": f"127.0.0.1:{self.rpcport}", + "monitoring_addr": f"127.0.0.1:{monitoring_port}", + } + self.conf_file = os.path.join(regtestdir, "electrs.toml") + with open(self.conf_file, "w") as f: + for k, v in electrs_conf.items(): + f.write(f'{k} = "{v}"\n') + + self.env = {"RUST_LOG": "DEBUG"} + + def start(self): + TailableProc.start(self) + logging.info("Electrs started") + + def startup(self): + try: + self.start() + except Exception: + self.stop() + raise + + def stop(self): + return TailableProc.stop(self) + + def cleanup(self): + try: + self.stop() + except Exception: + self.proc.kill() + self.proc.wait() + + def append_to_lianad_conf(self, conf_file): + with open(conf_file, "a") as f: + f.write("[electrum_config]\n") + f.write(f"addr = '127.0.0.1:{self.rpcport}'\n") diff --git a/tests/test_framework/lianad.py b/tests/test_framework/lianad.py index d9b4afdb6..5fcf9a38b 100644 --- a/tests/test_framework/lianad.py +++ b/tests/test_framework/lianad.py @@ -5,6 +5,8 @@ from bip380.descriptors import Descriptor from bip380.miniscript import SatisfactionMaterial from test_framework.utils import ( + BITCOIN_BACKEND_TYPE, + BitcoinBackendType, UnixDomainSocketRpc, TailableProc, VERBOSE, @@ -43,6 +45,7 @@ def __init__( self.cmd_line = [LIANAD_PATH, "--conf", f"{self.conf_file}"] socket_path = os.path.join(os.path.join(datadir, "regtest"), "lianad_rpc") self.rpc = UnixDomainSocketRpc(socket_path) + self.bitcoin_backend = bitcoin_backend with open(self.conf_file, "w") as f: f.write(f"data_dir = '{datadir}'\n") @@ -103,8 +106,9 @@ def restart_fresh(self, bitcoind): self.stop() dir_path = os.path.join(self.datadir, "regtest") shutil.rmtree(dir_path) - wallet_path = os.path.join(dir_path, "lianad_watchonly_wallet") - bitcoind.node_rpc.unloadwallet(wallet_path) + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + wallet_path = os.path.join(dir_path, "lianad_watchonly_wallet") + bitcoind.node_rpc.unloadwallet(wallet_path) self.start() wait_for( lambda: self.rpc.getinfo()["block_height"] == bitcoind.rpc.getblockcount() diff --git a/tests/test_framework/utils.py b/tests/test_framework/utils.py index d92f862f9..c48edc481 100644 --- a/tests/test_framework/utils.py +++ b/tests/test_framework/utils.py @@ -26,6 +26,7 @@ class BitcoinBackendType(str, enum.Enum): Bitcoind = "bitcoind" + Electrs = "electrs" DEFAULT_BITCOIN_BACKEND_TYPE = "bitcoind" @@ -34,6 +35,8 @@ class BitcoinBackendType(str, enum.Enum): ) DEFAULT_BITCOIND_PATH = "bitcoind" BITCOIND_PATH = os.getenv("BITCOIND_PATH", DEFAULT_BITCOIND_PATH) +DEFAULT_ELECTRS_PATH = "electrs" +ELECTRS_PATH = os.getenv("ELECTRS_PATH", DEFAULT_ELECTRS_PATH) OLD_LIANAD_PATH = os.getenv("OLD_LIANAD_PATH", None) IS_NOT_BITCOIND_24 = bool(int(os.getenv("IS_NOT_BITCOIND_24", True))) USE_TAPROOT = bool( diff --git a/tests/test_misc.py b/tests/test_misc.py index fecad7c3d..7cd1f95bb 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -7,6 +7,8 @@ from test_framework.authproxy import JSONRPCException from test_framework.serializations import PSBT from test_framework.utils import ( + BitcoinBackendType, + BITCOIN_BACKEND_TYPE, wait_for, RpcError, OLD_LIANAD_PATH, @@ -260,6 +262,10 @@ def test_coinbase_deposit(lianad, bitcoind): OLD_LIANAD_PATH is None or USE_TAPROOT, reason="Need the old lianad binary to create the datadir.", ) +@pytest.mark.skipif( + BITCOIN_BACKEND_TYPE is not BitcoinBackendType.Bitcoind, + reason="Only bitcoind backend was available for older lianad versions.", +) def test_migration(lianad_multisig, bitcoind): """Test we can start a newer lianad on a datadir created by an older lianad.""" lianad = lianad_multisig @@ -315,6 +321,10 @@ def bitcoind_wait_new_block(bitcoind): @pytest.mark.skipif( not IS_NOT_BITCOIND_24, reason="Need 'generateblock' with 'submit=False'" ) +@pytest.mark.skipif( + BITCOIN_BACKEND_TYPE is not BitcoinBackendType.Bitcoind, + reason="Tests the retry logic specific to the bitcoind backend.", +) def test_retry_on_workqueue_exceeded(lianad, bitcoind, executor): """Make sure we retry requests to bitcoind if it is temporarily overloaded.""" # Start by reducing the work queue to a single slot. Note we need to stop lianad diff --git a/tests/test_rpc.py b/tests/test_rpc.py index cd697aae1..dedc81bc4 100644 --- a/tests/test_rpc.py +++ b/tests/test_rpc.py @@ -686,11 +686,13 @@ def all_spent(coins): # descriptor. coins_before = sorted_coins() lianad.restart_fresh(bitcoind) - assert len(list_coins()) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(list_coins()) == 0 # The wallet isn't aware what derivation indexes were used. Necessarily it'll start # from 0. - assert lianad.rpc.getnewaddress() == first_address + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert lianad.rpc.getnewaddress() == first_address # Once the rescan is done, we must have detected all previous transactions. lianad.rpc.startrescan(initial_timestamp) From 72c63148984bc69ff7c4b04cfdc16a12994b2956 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Thu, 22 Aug 2024 14:18:49 +0100 Subject: [PATCH 13/17] ci: add electrs to func tests --- .cirrus.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 1487d84b1..384b67347 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -17,24 +17,47 @@ task: - USE_MIN_BITCOIN_VERSION: 'TRUE' - USE_MIN_BITCOIN_VERSION: 'FALSE' - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'electrs' + - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'electrs' - name: 'RPC functional tests' env: TEST_GROUP: tests/test_rpc.py matrix: - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'bitcoind' - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'electrs' + - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'electrs' - name: 'Chain functional tests' env: TEST_GROUP: tests/test_chain.py matrix: - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'electrs' - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'electrs' - name: 'Spend functional tests' env: TEST_GROUP: tests/test_spend.py matrix: - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'electrs' - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'electrs' cargo_registry_cache: folders: $CARGO_HOME/registry @@ -63,6 +86,7 @@ task: test_script: | set -xe + # We always need bitcoind, even when using a different backend. if [ "$USE_MIN_BITCOIN_VERSION" = "TRUE" ]; then # Download the minimum required bitcoind binary curl -O https://bitcoincore.org/bin/bitcoin-core-24.0.1/bitcoin-24.0.1-x86_64-linux-gnu.tar.gz @@ -78,6 +102,14 @@ task: export BITCOIND_PATH=bitcoin-26.0/bin/bitcoind fi + if [ "$BITCOIN_BACKEND_TYPE" = "electrs" ]; then + curl -OL https://github.com/RCasatta/electrsd/releases/download/electrs_releases/electrs_linux_v0.9.11.zip + echo "2b2f8aef35cd8e16e109b948a903d010aa472f6cdf2147d47e01fd95cd1785da electrs_linux_v0.9.11.zip" | sha256sum -c + unzip electrs_linux_v0.9.11.zip + chmod 754 electrs + export ELECTRS_PATH=$PWD/electrs + fi + # The misc tests have a backward compat test that need the path to a previous version of Liana. # For now it requires using 0.3. if [ "$TEST_GROUP" = "tests/test_misc.py" ]; then From 177dfc46c305ea7ab1c421e9a9d60535327fedff Mon Sep 17 00:00:00 2001 From: Michael Mallan Date: Tue, 27 Aug 2024 14:42:47 +0100 Subject: [PATCH 14/17] lib: expose BDK's electrum client --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 67b489fe8..b43ca31b2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ pub mod spend; #[cfg(test)] mod testutils; +pub use bdk_electrum::electrum_client; pub use bip39; use bitcoin::electrum; pub use miniscript; From ada89c5c56090338554cb8db0e078674798341ac Mon Sep 17 00:00:00 2001 From: Michael Mallan Date: Thu, 29 Aug 2024 10:07:39 +0100 Subject: [PATCH 15/17] doc: update with electrum info --- contrib/lianad_config_example.toml | 28 +++++++++++++++++++++++++--- doc/USAGE.md | 12 ++++++------ 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/contrib/lianad_config_example.toml b/contrib/lianad_config_example.toml index a18c96b16..b0edd609c 100644 --- a/contrib/lianad_config_example.toml +++ b/contrib/lianad_config_example.toml @@ -29,12 +29,34 @@ main_descriptor = "wsh(or_d(pk([0dd8c6f0/48'/1'/0'/2']tpubDFMbZ7U5k5hEfsttnZTKMm network = "testnet" poll_interval_secs = 30 -# This section is specific to the bitcoind implementation of the Bitcoin backend. This is the only -# implementation available for now. +# This section depends on the Bitcoin backend being used. +# +# If using bitcoind, the section name is [bitcoind_config]. # In order to be able to connect to bitcoind, it needs to know on what port it is listening and # how to authenticate, either by specifying the cookie location with "cookie_path" or otherwise # passing a colon-separated user and password with "auth". +# +# With cookie path: +# +# [bitcoind_config] +# addr = "127.0.0.1:18332" +# cookie_path = "/home/wizardsardine/.bitcoin/testnet3/.cookie" +# +# With user and password: +# +# [bitcoind_config] +# addr = "127.0.0.1:18332" +# auth = "my_user:my_password" +# +# +# If using an Electrum server, the section name is [electrum_config]. +# In order to connect, it needs the address as a string, which can be +# optionally prefixed with "ssl://" or "tcp://". If omitted, "tcp://" +# will be assumed. +# [electrum_config] +# addr = "127.0.0.1:50001" +# +# [bitcoind_config] addr = "127.0.0.1:18332" cookie_path = "/home/wizardsardine/.bitcoin/testnet3/.cookie" -# auth = "my_user:my_password" diff --git a/doc/USAGE.md b/doc/USAGE.md index b49f755fe..ca251b4ad 100644 --- a/doc/USAGE.md +++ b/doc/USAGE.md @@ -77,13 +77,13 @@ fear not! This is just a one time cost. Also, the full node is pruned so it will Liana can be run as a headless server using the `lianad` program. -As a Bitcoin wallet, Liana needs to be able to connect to the Bitcoin network. The software has been -developed such as multiple ways to connect to the Bitcoin network may be available. However for now -only the connection through `bitcoind` is implemented. +As a Bitcoin wallet, Liana needs to be able to connect to the Bitcoin network, +which is currently possible through the Bitcoin Core daemon (`bitcoind`) or an Electrum server. -Therefore in order to use Liana you need to have the Bitcoin Core daemon (`bitcoind`) running on your machine for the -desired network (mainnet, signet, testnet or regtest). The `bitcoind` installation may be pruned (note this may affect block chain -rescans) up to the maximum (around 550MB of blocks). +The chosen Bitcoin backend must be available while Liana is running. + +If using `bitcoind`, it must be running on your machine for the desired network (mainnet, signet, testnet or regtest) +and may be pruned (note this may affect block chain rescans) up to the maximum (around 550MB of blocks). The minimum supported version of Bitcoin Core is `24.0.1` (if you want to use Taproot it's `26.0`). If you don't have Bitcoin Core installed on your machine yet, you can download it From b630d46770319dce051c09133a528b1ff322a358 Mon Sep 17 00:00:00 2001 From: Michael Mallan Date: Mon, 2 Sep 2024 17:24:56 +0100 Subject: [PATCH 16/17] func test: wait for block heights to match This is copied from darosior's changes in https://github.com/wizardsardine/liana/pull/1222#issuecomment-2324894986. --- tests/test_rpc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_rpc.py b/tests/test_rpc.py index dedc81bc4..9d9aef1ec 100644 --- a/tests/test_rpc.py +++ b/tests/test_rpc.py @@ -902,6 +902,9 @@ def test_create_recovery(lianad, bitcoind): bitcoind.generate_block(9, wait_for_mempool=txid) # Now we can create a recovery tx that sweeps the first 3 coins. + wait_for( + lambda: lianad.rpc.getinfo()["block_height"] == bitcoind.rpc.getblockcount() + ) res = lianad.rpc.createrecovery(bitcoind.rpc.getnewaddress(), 18) reco_psbt = PSBT.from_base64(res["psbt"]) From 341f9406645dd26c7862ae7abe28aed5c7e9185d Mon Sep 17 00:00:00 2001 From: Michael Mallan Date: Thu, 5 Sep 2024 13:25:34 +0100 Subject: [PATCH 17/17] func test: prevent disconnects when using mocktime Thanks to pythcoiner for providing this fix. --- tests/test_framework/bitcoind.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_framework/bitcoind.py b/tests/test_framework/bitcoind.py index 95ae04578..a8a49b7ee 100644 --- a/tests/test_framework/bitcoind.py +++ b/tests/test_framework/bitcoind.py @@ -72,6 +72,12 @@ def __init__(self, bitcoin_dir, rpcport=None): "rpcport": rpcport, "fallbackfee": Decimal(1000) / COIN, "rpcthreads": 32, + # bitcoind uses mocktime in some tests, which can lead to peers (e.g. electrs) + # being disconnected. To prevent this, we set `peertimeout` greater than + # the max value being mocked. + # See https://github.com/bitcoin/bitcoin/blob/fa05ee0517d58b600f0ccad4c02c0734a23707d6/src/net.cpp#L1961. + # h/t pythcoiner :) + "peertimeout": 2 * 24 * 60 * 60, # 2 days } self.conf_file = os.path.join(bitcoin_dir, "bitcoin.conf") with open(self.conf_file, "w") as f: