diff --git a/.cirrus.yml b/.cirrus.yml index 1487d84b1..384b67347 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -17,24 +17,47 @@ task: - USE_MIN_BITCOIN_VERSION: 'TRUE' - USE_MIN_BITCOIN_VERSION: 'FALSE' - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'electrs' + - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'electrs' - name: 'RPC functional tests' env: TEST_GROUP: tests/test_rpc.py matrix: - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'bitcoind' - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'electrs' + - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'electrs' - name: 'Chain functional tests' env: TEST_GROUP: tests/test_chain.py matrix: - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'electrs' - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'electrs' - name: 'Spend functional tests' env: TEST_GROUP: tests/test_spend.py matrix: - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'bitcoind' + - USE_TAPROOT: 0 + BITCOIN_BACKEND_TYPE: 'electrs' - USE_TAPROOT: 1 + BITCOIN_BACKEND_TYPE: 'electrs' cargo_registry_cache: folders: $CARGO_HOME/registry @@ -63,6 +86,7 @@ task: test_script: | set -xe + # We always need bitcoind, even when using a different backend. if [ "$USE_MIN_BITCOIN_VERSION" = "TRUE" ]; then # Download the minimum required bitcoind binary curl -O https://bitcoincore.org/bin/bitcoin-core-24.0.1/bitcoin-24.0.1-x86_64-linux-gnu.tar.gz @@ -78,6 +102,14 @@ task: export BITCOIND_PATH=bitcoin-26.0/bin/bitcoind fi + if [ "$BITCOIN_BACKEND_TYPE" = "electrs" ]; then + curl -OL https://github.com/RCasatta/electrsd/releases/download/electrs_releases/electrs_linux_v0.9.11.zip + echo "2b2f8aef35cd8e16e109b948a903d010aa472f6cdf2147d47e01fd95cd1785da electrs_linux_v0.9.11.zip" | sha256sum -c + unzip electrs_linux_v0.9.11.zip + chmod 754 electrs + export ELECTRS_PATH=$PWD/electrs + fi + # The misc tests have a backward compat test that need the path to a previous version of Liana. # For now it requires using 0.3. if [ "$TEST_GROUP" = "tests/test_misc.py" ]; then diff --git a/Cargo.lock b/Cargo.lock index 335a18d52..91059d2b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,12 +62,32 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +[[package]] +name = "bdk_chain" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c601c4dc7e6c3efa538a0afbb43b964cefab9a9b5e8f352fa0ca38145448a5e7" +dependencies = [ + "bitcoin", + "miniscript", +] + [[package]] name = "bdk_coin_select" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c084bf76f0f67546fc814ffa82044144be1bb4618183a15016c162f8b087ad4" +[[package]] +name = "bdk_electrum" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28906275aeb1f71dc32045670f06c8a26fb17cc62151a99f7425d258f4bda589" +dependencies = [ + "bdk_chain", + "electrum-client", +] + [[package]] name = "bech32" version = "0.10.0-beta" @@ -139,6 +159,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cc" version = "1.0.83" @@ -172,7 +198,24 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "electrum-client" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89008f106be6f303695522f2f4c1f28b40c3e8367ed8b3bb227f1f882cb52cc2" +dependencies = [ + "bitcoin", + "byteorder", + "libc", + "log", + "rustls", + "serde", + "serde_json", + "webpki-roots", + "winapi", ] [[package]] @@ -268,6 +311,7 @@ version = "6.0.0" dependencies = [ "backtrace", "bdk_coin_select", + "bdk_electrum", "bip39", "dirs", "fern", @@ -438,6 +482,21 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rusqlite" version = "0.30.0" @@ -458,12 +517,44 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "secp256k1" version = "0.28.0" @@ -521,6 +612,12 @@ version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "syn" version = "2.0.46" @@ -591,6 +688,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "vcpkg" version = "0.2.15" @@ -609,13 +712,50 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -624,13 +764,29 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -639,42 +795,90 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "zerocopy" version = "0.7.32" diff --git a/Cargo.toml b/Cargo.toml index b3ce9f770..31fc8f63a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,10 @@ miniscript = { version = "11.0", features = ["serde", "compiler", "base64"] } # Coin selection algorithms for spend transaction creation. bdk_coin_select = "0.3" +# For Electrum backend. This is the latest version with the same bitcoin version as +# the miniscript dependency. +bdk_electrum = { version = "0.14" } + # Don't reinvent the wheel dirs = "5.0" diff --git a/contrib/lianad_config_example.toml b/contrib/lianad_config_example.toml index a18c96b16..b0edd609c 100644 --- a/contrib/lianad_config_example.toml +++ b/contrib/lianad_config_example.toml @@ -29,12 +29,34 @@ main_descriptor = "wsh(or_d(pk([0dd8c6f0/48'/1'/0'/2']tpubDFMbZ7U5k5hEfsttnZTKMm network = "testnet" poll_interval_secs = 30 -# This section is specific to the bitcoind implementation of the Bitcoin backend. This is the only -# implementation available for now. +# This section depends on the Bitcoin backend being used. +# +# If using bitcoind, the section name is [bitcoind_config]. # In order to be able to connect to bitcoind, it needs to know on what port it is listening and # how to authenticate, either by specifying the cookie location with "cookie_path" or otherwise # passing a colon-separated user and password with "auth". +# +# With cookie path: +# +# [bitcoind_config] +# addr = "127.0.0.1:18332" +# cookie_path = "/home/wizardsardine/.bitcoin/testnet3/.cookie" +# +# With user and password: +# +# [bitcoind_config] +# addr = "127.0.0.1:18332" +# auth = "my_user:my_password" +# +# +# If using an Electrum server, the section name is [electrum_config]. +# In order to connect, it needs the address as a string, which can be +# optionally prefixed with "ssl://" or "tcp://". If omitted, "tcp://" +# will be assumed. +# [electrum_config] +# addr = "127.0.0.1:50001" +# +# [bitcoind_config] addr = "127.0.0.1:18332" cookie_path = "/home/wizardsardine/.bitcoin/testnet3/.cookie" -# auth = "my_user:my_password" diff --git a/doc/USAGE.md b/doc/USAGE.md index b49f755fe..ca251b4ad 100644 --- a/doc/USAGE.md +++ b/doc/USAGE.md @@ -77,13 +77,13 @@ fear not! This is just a one time cost. Also, the full node is pruned so it will Liana can be run as a headless server using the `lianad` program. -As a Bitcoin wallet, Liana needs to be able to connect to the Bitcoin network. The software has been -developed such as multiple ways to connect to the Bitcoin network may be available. However for now -only the connection through `bitcoind` is implemented. +As a Bitcoin wallet, Liana needs to be able to connect to the Bitcoin network, +which is currently possible through the Bitcoin Core daemon (`bitcoind`) or an Electrum server. -Therefore in order to use Liana you need to have the Bitcoin Core daemon (`bitcoind`) running on your machine for the -desired network (mainnet, signet, testnet or regtest). The `bitcoind` installation may be pruned (note this may affect block chain -rescans) up to the maximum (around 550MB of blocks). +The chosen Bitcoin backend must be available while Liana is running. + +If using `bitcoind`, it must be running on your machine for the desired network (mainnet, signet, testnet or regtest) +and may be pruned (note this may affect block chain rescans) up to the maximum (around 550MB of blocks). The minimum supported version of Bitcoin Core is `24.0.1` (if you want to use Taproot it's `26.0`). If you don't have Bitcoin Core installed on your machine yet, you can download it diff --git a/src/bitcoin/d/mod.rs b/src/bitcoin/d/mod.rs index 41e25216a..fe26b7327 100644 --- a/src/bitcoin/d/mod.rs +++ b/src/bitcoin/d/mod.rs @@ -1096,7 +1096,7 @@ impl BitcoinD { } pub fn start_rescan( - &self, + &mut self, desc: &LianaDescriptor, timestamp: u32, ) -> Result<(), BitcoindError> { diff --git a/src/bitcoin/electrum/client.rs b/src/bitcoin/electrum/client.rs new file mode 100644 index 000000000..230adcf6e --- /dev/null +++ b/src/bitcoin/electrum/client.rs @@ -0,0 +1,411 @@ +use std::{collections::HashSet, convert::TryInto}; + +use bdk_electrum::{ + bdk_chain::{ + bitcoin, + local_chain::{CheckPoint, LocalChain}, + spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult}, + BlockId, ChainPosition, ConfirmationHeightAnchor, TxGraph, + }, + electrum_client::{self, Config, ElectrumApi}, + ElectrumExt, +}; + +use super::utils::{ + block_id_from_tip, height_i32_from_usize, height_usize_from_i32, outpoints_from_tx, +}; +use crate::{ + bitcoin::{electrum::utils::tip_from_block_id, BlockChainTip, MempoolEntry, MempoolEntryFees}, + config, +}; + +// Default batch size to use when making requests to the Electrum server. +const DEFAULT_BATCH_SIZE: usize = 200; + +// If Electrum takes more than 3 minutes to answer one of our queries, fail. +const RPC_SOCKET_TIMEOUT: u8 = 180; + +// Number of retries while communicating with the Electrum server. +// A retry happens with exponential back-off (base 2) so this makes us give up after (1+2+4+8+16+32=) 63 seconds. +const RETRY_LIMIT: u8 = 6; + +/// An error in the Electrum client. +#[derive(Debug)] +pub enum Error { + Server(electrum_client::Error), + TipChanged(BlockId, BlockId), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Error::Server(e) => write!(f, "Electrum error: '{}'.", e), + Error::TipChanged(expected, actual) => write!( + f, + "Electrum error: Expected tip '{}' but actual tip was {}.", + tip_from_block_id(*expected), + tip_from_block_id(*actual), + ), + } + } +} + +pub struct Client(electrum_client::Client); + +impl Client { + /// Create a new client and perform sanity checks. + pub fn new(electrum_config: &config::ElectrumConfig) -> Result { + let config = Config::builder() + .retry(RETRY_LIMIT) + .timeout(Some(RPC_SOCKET_TIMEOUT)) + .build(); + let client = + bdk_electrum::electrum_client::Client::from_config(&electrum_config.addr, config) + .map_err(Error::Server)?; + Ok(Self(client)) + } + + pub fn chain_tip(&self) -> Result { + self.0 + .block_headers_subscribe() + .map_err(Error::Server) + .map(|notif| BlockChainTip { + height: height_i32_from_usize(notif.height), + hash: notif.header.block_hash(), + }) + } + + fn genesis_block_header(&self) -> Result { + self.0.block_header(0).map_err(Error::Server) + } + + pub fn genesis_block_timestamp(&self) -> Result { + self.genesis_block_header().map(|header| header.time) + } + + pub fn genesis_block(&self) -> Result { + self.genesis_block_header().map(|header| BlockChainTip { + hash: header.block_hash(), + height: 0, + }) + } + + pub fn broadcast_tx(&self, tx: &bitcoin::Transaction) -> Result { + self.0.transaction_broadcast(tx).map_err(Error::Server) + } + + pub fn tip_time(&self) -> Result { + let tip_height = self.chain_tip()?.height; + self.0 + .block_header(height_usize_from_i32(tip_height)) + .map_err(Error::Server) + .map(|bh| bh.time) + } + + fn sync_with_confirmation_height_anchor( + &self, + request: SyncRequest, + fetch_prev_txouts: bool, + ) -> Result, Error> { + Ok(self + .0 + .sync(request, DEFAULT_BATCH_SIZE, fetch_prev_txouts) + .map_err(Error::Server)? + .with_confirmation_height_anchor()) + } + + /// Perform the given `SyncRequest` with `ConfirmationTimeHeightAnchor`. + pub fn sync_with_confirmation_time_height_anchor( + &self, + request: SyncRequest, + fetch_prev_txouts: bool, + ) -> Result { + self.0 + .sync(request, DEFAULT_BATCH_SIZE, fetch_prev_txouts) + .map_err(Error::Server)? + .with_confirmation_time_height_anchor(&self.0) + .map_err(Error::Server) + } + + /// Perform the given `FullScanRequest` with `ConfirmationTimeHeightAnchor`. + pub fn full_scan_with_confirmation_time_height_anchor( + &self, + request: FullScanRequest, + stop_gap: usize, + fetch_prev_txouts: bool, + ) -> Result, Error> { + self.0 + .full_scan(request, stop_gap, DEFAULT_BATCH_SIZE, fetch_prev_txouts) + .map_err(Error::Server)? + .with_confirmation_time_height_anchor(&self.0) + .map_err(Error::Server) + } + + /// Get mempool entries. + /// + /// If `expected_tip` is specified, the function will return `Error::TipChanged` if the chain tip + /// changes while the entries are being found. Otherwise, the function will restart in case the + /// chain tip changes before completion. + fn mempool_entries( + &self, + txids: HashSet, + expected_tip: Option, + ) -> Result, Error> { + log::debug!("Getting mempool entries for txids '{:?}'.", txids); + let mut graph = TxGraph::default(); + let mut local_chain = LocalChain::from_genesis_hash(self.genesis_block()?.hash).0; + let tip_block = if let Some(ref expected_tip) = expected_tip { + expected_tip.block_id() + } else { + block_id_from_tip(self.chain_tip()?) + }; + if tip_block.height > 0 { + let _ = local_chain + .insert_block(tip_block) + .expect("only contains genesis block"); + } + // First, get the tx itself and check it's unconfirmed. + let request = SyncRequest::from_chain_tip(local_chain.tip()).chain_txids(txids.clone()); + // We'll get prev txouts for this tx when we find its ancestors below. + let sync_result = self.sync_with_confirmation_height_anchor(request, false)?; + let _ = local_chain.apply_update(sync_result.chain_update); + // Store local tip after first sync. This will be our reference tip. + let local_tip = local_chain.tip(); + if let Some(ref expected_tip) = expected_tip { + if expected_tip != &local_chain.tip() { + return Err(Error::TipChanged( + expected_tip.block_id(), + local_chain.tip().block_id(), + )); + } + } + let mut desc_ops = Vec::new(); + let mut txs = Vec::new(); + for txid in &txids { + if let Some(ChainPosition::Unconfirmed(_)) = sync_result + .graph_update + .get_chain_position(&local_chain, local_chain.tip().block_id(), *txid) + { + let tx = sync_result + .graph_update + .get_tx(*txid) + .expect("we must have tx in graph after sync"); + desc_ops.extend(outpoints_from_tx(&tx)); + txs.push(tx); + } + } + let _ = graph.apply_update(sync_result.graph_update); + // Now iterate over increasing depths of descendants. + // As they are descendants, we can assume they are all unconfirmed. + while !desc_ops.is_empty() { + log::debug!("Syncing descendant outpoints: {:?}", desc_ops); + let request = SyncRequest::from_chain_tip(local_chain.tip()) + .cache_graph_txs(&graph) + .chain_outpoints(desc_ops.clone()); + // Fetch prev txouts to ensure we have all required txs in the graph to calculate fees. + // An unconfirmed descendant may have a confirmed parent that we wouldn't have in our graph. + let sync_result = self.sync_with_confirmation_height_anchor(request, true)?; + let _ = local_chain.apply_update(sync_result.chain_update); + if let Some(ref expected_tip) = expected_tip { + if expected_tip != &local_chain.tip() { + return Err(Error::TipChanged( + expected_tip.block_id(), + local_chain.tip().block_id(), + )); + } + } + if local_chain.tip() != local_tip { + log::debug!("Chain tip changed while getting mempool entry. Restarting."); + return self.mempool_entries(txids, expected_tip.clone()); + } + let _ = graph.apply_update(sync_result.graph_update); + // Get any txids spending the outpoints we've just synced against. + let desc_txids: HashSet<_> = graph + .filter_chain_txouts( + &local_chain, + local_chain.tip().block_id(), + desc_ops.iter().map(|op| ((), *op)), + ) + .filter_map(|(_, txout)| txout.spent_by.map(|(_, spend_txid)| spend_txid)) + .collect(); + desc_ops = desc_txids + .iter() + .flat_map(|txid| { + let desc_tx = graph + .get_tx(*txid) + .expect("we must have tx in graph after sync"); + outpoints_from_tx(&desc_tx) + }) + .collect(); + } + + // For each unconfirmed transaction, starting with `txid`, get its direct ancestors, which may be confirmed or unconfirmed. + // Continue until there are no more unconfirmed ancestors. + // Confirmed transactions will be filtered out from `anc_txids` later on. + let mut anc_txids: HashSet<_> = txs + .iter() + .flat_map(|tx| tx.input.iter().map(|txin| txin.previous_output.txid)) + .collect(); + while !anc_txids.is_empty() { + log::debug!("Syncing ancestor txids: {:?}", anc_txids); + let request = SyncRequest::from_chain_tip(local_chain.tip()) + .cache_graph_txs(&graph) + .chain_txids(anc_txids.clone()); + // We expect to have prev txouts for all unconfirmed ancestors in our graph so no need to fetch them here. + // Note we keep iterating through ancestors until we find one that is confirmed and only need to calculate + // fees for unconfirmed transactions. + let sync_result = self.sync_with_confirmation_height_anchor(request, false)?; + let _ = local_chain.apply_update(sync_result.chain_update); + if let Some(expected_tip) = &expected_tip { + if expected_tip != &local_chain.tip() { + return Err(Error::TipChanged( + expected_tip.block_id(), + local_chain.tip().block_id(), + )); + } + } + if local_chain.tip() != local_tip { + log::debug!("Chain tip changed while getting mempool entry. Restarting."); + return self.mempool_entries(txids, expected_tip); + } + let _ = graph.apply_update(sync_result.graph_update); + + // Add ancestors of any unconfirmed txs. + anc_txids = anc_txids + .iter() + .filter_map(|anc_txid| { + if let Some(ChainPosition::Unconfirmed(_)) = graph.get_chain_position( + &local_chain, + local_chain.tip().block_id(), + *anc_txid, + ) { + let anc_tx = graph.get_tx(*anc_txid).expect("we must have it"); + Some( + anc_tx + .input + .clone() + .iter() + .map(|txin| txin.previous_output.txid) + .collect::>(), + ) + } else { + None + } + }) + .flatten() + .collect(); + } + let mut entries = Vec::new(); + for tx in txs { + // Now iterate over ancestors and descendants in the graph. + let base_fee = graph + .calculate_fee(&tx) + .expect("all required txs are in graph"); + let base_size = tx.vsize(); + // Ancestor & descendant fees include those of `txid`. + let mut desc_fees = base_fee; + let mut anc_fees = base_fee; + // Ancestor size includes that of `txid`. + let mut anc_size = base_size; + for desc_txid in graph.walk_descendants(tx.txid(), |_, desc_txid| Some(desc_txid)) { + log::debug!("Getting fee for desc txid '{}'.", desc_txid); + let desc_tx = graph + .get_tx(desc_txid) + .expect("all descendant txs are in graph"); + let fee = graph + .calculate_fee(&desc_tx) + .expect("all required txs are in graph"); + desc_fees += fee; + } + for anc_tx in graph.walk_ancestors(tx, |_, anc_tx| Some(anc_tx)) { + log::debug!("Getting fee and size for anc txid '{}'.", anc_tx.txid()); + if let Some(ChainPosition::Unconfirmed(_)) = graph.get_chain_position( + &local_chain, + local_chain.tip().block_id(), + anc_tx.txid(), + ) { + let fee = graph + .calculate_fee(&anc_tx) + .expect("all required txs are in graph"); + anc_fees += fee; + anc_size += anc_tx.vsize(); + } else { + log::debug!("Ancestor txid '{}' is not unconfirmed.", anc_tx.txid()); + continue; + } + } + let fees = MempoolEntryFees { + base: bitcoin::Amount::from_sat(base_fee), + ancestor: bitcoin::Amount::from_sat(anc_fees), + descendant: bitcoin::Amount::from_sat(desc_fees), + }; + let entry = MempoolEntry { + vsize: base_size.try_into().expect("tx size must fit into u64"), + fees, + ancestor_vsize: anc_size.try_into().expect("tx size must fit into u64"), + }; + entries.push(entry) + } + + // It's possible that the chain tip has now changed, but it hadn't done as of the last sync, + // so go ahead and return the results. + Ok(entries) + } + + /// Get mempool entry for a single `txid`. + /// + /// Convenience method to call `mempool_entries` for a single `txid`, + /// returning `Option` instead of `Vec`. + pub fn mempool_entry(&self, txid: &bitcoin::Txid) -> Result, Error> { + // We just require the chain tip to stay the same while running `mempool_entries` so + // don't need to pass in an expected tip. + self.mempool_entries(HashSet::from([*txid]), None) + .map(|entries| entries.first().cloned()) + } + + /// Get mempool spenders of the given outpoints. + /// + /// Will restart if chain tip changes before completion. + pub fn mempool_spenders( + &self, + outpoints: &[bitcoin::OutPoint], + ) -> Result, Error> { + log::debug!("Getting mempool spenders for outpoints: {:?}.", outpoints); + let mut local_chain = LocalChain::from_genesis_hash(self.genesis_block()?.hash).0; + let chain_tip = self.chain_tip()?; + if chain_tip.height > 0 { + let _ = local_chain + .insert_block(block_id_from_tip(chain_tip)) + .expect("only contains genesis block"); + } + let request = + SyncRequest::from_chain_tip(local_chain.tip()).chain_outpoints(outpoints.to_vec()); + // We don't need to fetch prev txouts as we just want the outspends. + let sync_result = self.sync_with_confirmation_height_anchor(request, false)?; + let _ = local_chain.apply_update(sync_result.chain_update); + // Store tip at which first sync was completed. This will be our reference tip. + let local_tip = local_chain.tip(); + let graph = sync_result.graph_update; + let txids: HashSet<_> = outpoints + .iter() + .flat_map(|op| graph.outspends(*op)) + .copied() + .collect(); + let entries = match self.mempool_entries(txids, Some(local_tip)) { + Ok(entries) => entries, + Err(Error::TipChanged(expected, actual)) => { + log::debug!( + "Chain tip changed from {:?} to {:?} while \ + getting mempool spenders. Restarting.", + expected, + actual + ); + return self.mempool_spenders(outpoints); + } + Err(e) => { + return Err(e); + } + }; + Ok(entries) + } +} diff --git a/src/bitcoin/electrum/mod.rs b/src/bitcoin/electrum/mod.rs new file mode 100644 index 000000000..8b9745c76 --- /dev/null +++ b/src/bitcoin/electrum/mod.rs @@ -0,0 +1,250 @@ +use std::collections::HashMap; + +use bdk_electrum::bdk_chain::{ + bitcoin::{self, bip32::ChildNumber, BlockHash, OutPoint}, + local_chain::LocalChain, + spk_client::{FullScanRequest, SyncRequest}, + ChainPosition, +}; + +pub mod client; +mod utils; +pub mod wallet; +use crate::bitcoin::{Block, BlockChainTip, Coin}; + +/// An error in the Electrum interface. +#[derive(Debug)] +pub enum ElectrumError { + Client(client::Error), + GenesisHashMismatch( + BlockHash, /*expected hash*/ + BlockHash, /*server hash*/ + BlockHash, /*wallet hash*/ + ), +} + +impl std::fmt::Display for ElectrumError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + ElectrumError::Client(e) => write!(f, "Electrum client error: '{}'.", e), + ElectrumError::GenesisHashMismatch(expected, server, wallet) => { + write!( + f, + "Genesis hash mismatch. The genesis hash is expected to be '{}'. \ + The server has hash '{}' and the wallet has hash '{}'.", + expected, server, wallet, + ) + } + } + } +} + +/// Interface for Electrum backend. +pub struct Electrum { + client: client::Client, + bdk_wallet: wallet::BdkWallet, + /// Used for setting the `last_seen` of unconfirmed transactions in a strictly + /// increasing manner. + sync_count: u64, + /// Set to `true` to force a full scan from the genesis block regardless of + /// the wallet's local chain height. + full_scan: bool, +} + +impl Electrum { + pub fn new( + client: client::Client, + bdk_wallet: wallet::BdkWallet, + ) -> Result { + Ok(Self { + client, + bdk_wallet, + sync_count: 0, + full_scan: false, // by default, only perform full scan if wallet's local chain has height 0 + }) + } + + pub fn sanity_checks(&self, expected_hash: &bitcoin::BlockHash) -> Result<(), ElectrumError> { + let server_hash = self + .client + .genesis_block() + .map_err(ElectrumError::Client)? + .hash; + let wallet_hash = self.bdk_wallet.local_chain().genesis_hash(); + if server_hash != *expected_hash || wallet_hash != *expected_hash { + return Err(ElectrumError::GenesisHashMismatch( + *expected_hash, + server_hash, + wallet_hash, + )); + } + Ok(()) + } + + pub fn client(&self) -> &client::Client { + &self.client + } + + fn local_chain(&self) -> &LocalChain { + self.bdk_wallet.local_chain() + } + + /// Get all coins stored in the wallet, taking into consideration only those unconfirmed + /// transactions that were seen in the last wallet sync. + pub fn wallet_coins(&self, outpoints: Option<&[OutPoint]>) -> HashMap { + self.bdk_wallet.coins(outpoints, Some(self.sync_count)) + } + + /// Get the tip of the wallet's local chain. + pub fn wallet_tip(&self) -> BlockChainTip { + utils::tip_from_block_id(self.local_chain().tip().block_id()) + } + + /// Whether `tip` exists in the wallet's `local_chain`. + /// + /// Returns `None` if no block at that height exists in `local_chain`. + pub fn is_in_wallet_chain(&self, tip: BlockChainTip) -> Option { + self.bdk_wallet.is_in_chain(tip) + } + + /// Whether we'll perform a full scan at the next poll. + pub fn is_rescanning(&self) -> bool { + self.full_scan || self.local_chain().tip().height() == 0 + } + + /// Make the poller perform a full scan on the next iteration. + pub fn trigger_rescan(&mut self) { + self.full_scan = true; + } + + /// Sync the wallet with the Electrum server. If there was any reorg since the last poll, this + /// returns the first common ancestor between the previous and the new chain. + pub fn sync_wallet( + &mut self, + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Result, ElectrumError> { + self.bdk_wallet.reveal_spks(receive_index, change_index); + let local_chain_tip = self.local_chain().tip(); + log::debug!( + "local chain tip height before sync with electrum: {}", + local_chain_tip.block_id().height + ); + + // We'll only need to calculate fees of mempool transactions and this will be done separately from our graph + // so we don't need to fetch prev txouts. In any case, we'll already have these for our own transactions. + const FETCH_PREV_TXOUTS: bool = false; + const STOP_GAP: usize = 50; + + let (chain_update, mut graph_update, keychain_update) = if !self.is_rescanning() { + log::info!("Performing sync."); + let mut request = SyncRequest::from_chain_tip(local_chain_tip.clone()) + .cache_graph_txs(self.bdk_wallet.graph()); + + let all_spks: Vec<_> = self + .bdk_wallet + .index() + .inner() // we include lookahead SPKs + .all_spks() + .iter() + .map(|(_, script)| script.clone()) + .collect(); + request = request.chain_spks(all_spks); + log::debug!("num SPKs for sync: {}", request.spks.len()); + + let sync_result = self + .client + .sync_with_confirmation_time_height_anchor(request, FETCH_PREV_TXOUTS) + .map_err(ElectrumError::Client)?; + log::info!("Sync complete."); + (sync_result.chain_update, sync_result.graph_update, None) + } else { + log::info!("Performing full scan."); + // Either local_chain has height 0 or we want to trigger a full scan. + // In both cases, the scan should be from the genesis block. + let genesis_block = local_chain_tip.get(0).expect("must contain genesis block"); + let mut request = FullScanRequest::from_chain_tip(genesis_block) + .cache_graph_txs(self.bdk_wallet.graph()); + + for (k, spks) in self.bdk_wallet.index().all_unbounded_spk_iters() { + request = request.set_spks_for_keychain(k, spks); + } + let scan_result = self + .client + .full_scan_with_confirmation_time_height_anchor( + request, + STOP_GAP, + FETCH_PREV_TXOUTS, + ) + .map_err(ElectrumError::Client)?; + // A full scan only makes sense to do once, in most cases. Don't do it again unless + // explicitly asked to by a user. + self.full_scan = false; + log::info!("Full scan complete."); + ( + scan_result.chain_update, + scan_result.graph_update, + Some(scan_result.last_active_indices), + ) + }; + log::debug!( + "chain update height after sync with electrum: {}", + chain_update.height() + ); + + // Increment the sync count and apply changes. + self.sync_count = self.sync_count.checked_add(1).expect("must fit"); + if let Some(keychain_update) = keychain_update { + self.bdk_wallet.apply_keychain_update(keychain_update); + } + let changeset = self.bdk_wallet.apply_connected_chain_update(chain_update); + + let mut changes_iter = changeset.into_iter(); + let reorg_common_ancestor = loop { + match changes_iter.next() { + Some((height, Some(_))) => { + // `BlockHash` being `Some(_)` means a checkpoint at this height was added to the chain. + // Since we iterate in ascending height order, we'll see the lowest block height first. + // If the lowest height it adds is higher than our height before syncing, we're good. + // Else if it's adding a block at height before syncing or lower, it's a reorg. + break if height > local_chain_tip.height() { + None + } else { + log::info!("Block chain reorganization detected."); + Some(self.bdk_wallet.find_block_at_or_before_height(height)) + }; + } + Some((_, None)) => continue, + None => break None, + } + }; + + // Unconfirmed transactions have their last seen as 0, so we override to the `sync_count` + // so that conflicts can be properly handled. We use `sync_count` instead of current time + // in seconds to ensure strictly increasing values between poller iterations. + for tx in &graph_update.initial_changeset().txs { + let txid = tx.txid(); + if let Some(ChainPosition::Unconfirmed(_)) = graph_update.get_chain_position( + self.local_chain(), + self.local_chain().tip().block_id(), + txid, + ) { + log::debug!( + "changing last seen for txid '{}' to {}", + txid, + self.sync_count + ); + let _ = graph_update.insert_seen_at(txid, self.sync_count); + } + } + self.bdk_wallet.apply_graph_update(graph_update); + Ok(reorg_common_ancestor) + } + + pub fn wallet_transaction( + &self, + txid: &bitcoin::Txid, + ) -> Option<(bitcoin::Transaction, Option)> { + self.bdk_wallet.get_transaction(txid) + } +} diff --git a/src/bitcoin/electrum/utils.rs b/src/bitcoin/electrum/utils.rs new file mode 100644 index 000000000..2da51983b --- /dev/null +++ b/src/bitcoin/electrum/utils.rs @@ -0,0 +1,55 @@ +use std::convert::TryInto; + +use bdk_electrum::bdk_chain::{bitcoin, BlockId, ConfirmationTimeHeightAnchor}; + +use crate::bitcoin::{BlockChainTip, BlockInfo}; + +pub fn height_u32_from_i32(height: i32) -> u32 { + height.try_into().expect("height must fit into u32") +} + +pub fn height_i32_from_u32(height: u32) -> i32 { + height.try_into().expect("height must fit into i32") +} + +pub fn height_i32_from_usize(height: usize) -> i32 { + height.try_into().expect("height must fit into i32") +} + +pub fn height_usize_from_i32(height: i32) -> usize { + height.try_into().expect("height must fit into usize") +} + +pub fn block_id_from_tip(tip: BlockChainTip) -> BlockId { + BlockId { + height: height_u32_from_i32(tip.height), + hash: tip.hash, + } +} + +pub fn tip_from_block_id(id: BlockId) -> BlockChainTip { + BlockChainTip { + height: height_i32_from_u32(id.height), + hash: id.hash, + } +} + +pub fn block_info_from_anchor(anchor: ConfirmationTimeHeightAnchor) -> BlockInfo { + BlockInfo { + height: height_i32_from_u32(anchor.confirmation_height), + time: anchor + .confirmation_time + .try_into() + .expect("u32 by consensus"), + } +} + +/// Get the transaction's outpoints. +pub fn outpoints_from_tx(tx: &bitcoin::Transaction) -> Vec { + let txid = tx.txid(); + (0..tx.output.len()) + .map(|i| { + bitcoin::OutPoint::new(txid, i.try_into().expect("num tx outputs must fit in u32")) + }) + .collect::>() +} diff --git a/src/bitcoin/electrum/wallet.rs b/src/bitcoin/electrum/wallet.rs new file mode 100644 index 000000000..2e467f214 --- /dev/null +++ b/src/bitcoin/electrum/wallet.rs @@ -0,0 +1,328 @@ +use std::{ + collections::{BTreeMap, HashMap}, + convert::TryInto, + sync::Arc, +}; + +use bdk_electrum::bdk_chain::{ + bitcoin::{self, bip32, BlockHash, OutPoint, ScriptBuf, TxOut}, + keychain::KeychainTxOutIndex, + local_chain::{ChangeSet as ChainChangeSet, CheckPoint, LocalChain}, + miniscript::{Descriptor, DescriptorPublicKey}, + tx_graph::{self, TxGraph}, + ChainOracle, ChainPosition, ConfirmationTimeHeightAnchor, IndexedTxGraph, +}; +use miniscript::bitcoin::bip32::ChildNumber; + +use super::utils::{ + block_id_from_tip, block_info_from_anchor, height_i32_from_u32, height_u32_from_i32, +}; +use crate::{ + bitcoin::{Block, BlockChainTip, Coin, COINBASE_MATURITY}, + descriptors::LianaDescriptor, +}; + +// TODO: Move and reuse `liana::database::sqlite::utils::LOOK_AHEAD_LIMIT`? +const LOOK_AHEAD_LIMIT: u32 = 200; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum KeychainType { + Receive, + Change, +} + +pub struct BdkWallet { + graph: IndexedTxGraph>, + local_chain: LocalChain, + // Store descriptors for use when getting SPKs. + receive_desc: Descriptor, + change_desc: Descriptor, +} + +impl BdkWallet { + /// Create a new BDK wallet and initialize with the given data that was + /// valid as of `tip`. + /// + /// If there is no `tip`, then any provided data will be ignored. + /// + /// `receive_index` and `change_index` are the last used derivation + /// indices for the receive and change descriptors, respectively. + pub fn new( + main_descriptor: &LianaDescriptor, + genesis_hash: BlockHash, + tip: Option, + coins: &[Coin], + txs: &[bitcoin::Transaction], + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Self { + let local_chain = LocalChain::from_genesis_hash(genesis_hash).0; + let receive_desc = main_descriptor + .receive_descriptor() + .as_descriptor_public_key(); + let change_desc = main_descriptor + .change_descriptor() + .as_descriptor_public_key(); + + let mut bdk_wallet = BdkWallet { + graph: { + let mut indexer = KeychainTxOutIndex::::new(LOOK_AHEAD_LIMIT); + let _ = indexer.insert_descriptor(KeychainType::Receive, receive_desc.clone()); + let _ = indexer.insert_descriptor(KeychainType::Change, change_desc.clone()); + IndexedTxGraph::new(indexer) + }, + local_chain, + receive_desc: receive_desc.clone(), + change_desc: change_desc.clone(), + }; + if let Some(tip) = tip { + // This will be our anchor for any confirmed transactions. + let anchor_block = block_id_from_tip(tip); + if tip.height > 0 { + log::debug!("inserting block into local chain: {:?}", anchor_block); + let _ = bdk_wallet + .local_chain + .insert_block(anchor_block) + .expect("local chain only contains genesis block"); + } + // Update the last used derivation index for both change and receive addresses. + log::debug!( + "revealing SPKs up to receive index {receive_index} and change index {change_index}" + ); + bdk_wallet.reveal_spks(receive_index, change_index); + + // Update the existing coins and transactions information using a TxGraph changeset. + log::debug!("Number of coins to load: {}.", coins.len()); + log::debug!("Number of txs to load: {}.", txs.len()); + let mut graph_cs = tx_graph::ChangeSet::default(); + for tx in txs { + graph_cs.txs.insert(Arc::new(tx.clone())); + } + for coin in coins { + // First of all insert the txout itself. + let script_pubkey = bdk_wallet.get_spk(coin.derivation_index, coin.is_change); + let txout = TxOut { + script_pubkey, + value: coin.amount, + }; + graph_cs.txouts.insert(coin.outpoint, txout); + // If the coin's deposit transaction is confirmed, tell BDK by inserting an anchor. + // Otherwise, we could insert a last seen timestamp but we don't have such data stored in + // the table. + if let Some(block) = coin.block_info { + graph_cs.anchors.insert(( + ConfirmationTimeHeightAnchor { + confirmation_height: height_u32_from_i32(block.height), + confirmation_time: block.time.into(), + anchor_block, + }, + coin.outpoint.txid, + )); + } + // If the coin's spending transaction is confirmed, do the same. + if let Some(block) = coin.spend_block { + let spend_txid = coin.spend_txid.expect("Must be present if confirmed."); + graph_cs.anchors.insert(( + ConfirmationTimeHeightAnchor { + confirmation_height: height_u32_from_i32(block.height), + confirmation_time: block.time.into(), + anchor_block, + }, + spend_txid, + )); + } + } + let mut graph = TxGraph::default(); + graph.apply_changeset(graph_cs); + let _ = bdk_wallet.graph.apply_update(graph); + } + bdk_wallet + } + + /// Get a reference to the local chain. + pub fn local_chain(&self) -> &LocalChain { + &self.local_chain + } + + /// Whether `tip` exists in `local_chain`. + /// + /// Returns `None` if no block at that height exists in `local_chain`. + pub fn is_in_chain(&self, tip: BlockChainTip) -> Option { + self.local_chain + .is_block_in_chain(block_id_from_tip(tip), self.local_chain().tip().block_id()) + .expect("function is infallible") + } + + /// Get a reference to the graph. + pub fn graph(&self) -> &TxGraph { + self.graph.graph() + } + + /// Get a reference to the transaction index. + pub fn index(&self) -> &KeychainTxOutIndex { + &self.graph.index + } + + /// Reveal SPKs based on derivation indices set in DB. + pub fn reveal_spks(&mut self, receive_index: ChildNumber, change_index: ChildNumber) { + let mut keychain_update = BTreeMap::new(); + keychain_update.insert(KeychainType::Receive, receive_index.into()); + keychain_update.insert(KeychainType::Change, change_index.into()); + self.apply_keychain_update(keychain_update) + } + + fn get_spk(&self, der_index: bip32::ChildNumber, is_change: bool) -> ScriptBuf { + // Try to get it from the BDK wallet cache first, failing that derive it from the appropriate + // descriptor. + let chain_kind = if is_change { + KeychainType::Change + } else { + KeychainType::Receive + }; + if let Some(spk) = self.graph.index.spk_at_index(chain_kind, der_index.into()) { + spk.to_owned() + } else { + let desc = if is_change { + &self.change_desc + } else { + &self.receive_desc + }; + desc.at_derivation_index(der_index.into()) + .expect("Not multipath and index isn't hardened.") + .script_pubkey() + } + } + + /// Get the coins currently stored by the `BdkWallet` optionally filtered by `outpoints`. + /// If `outpoints` is `None`, no filter will be applied. + /// If `outpoints` is an empty slice, no coins will be returned. + /// If `last_seen` is set, only those unconfirmed transactions with a matching last seen + /// will be considered. + pub fn coins( + &self, + outpoints: Option<&[bitcoin::OutPoint]>, + last_seen: Option, + ) -> HashMap { + // Get an iterator over all the wallet txos (not only the currently unspent ones) by using + // lower level methods. + let tx_graph = self.graph.graph(); + let txo_index = &self.graph.index; + let tip_id = self.local_chain.tip().block_id(); + let wallet_txos = + tx_graph.filter_chain_txouts(&self.local_chain, tip_id, txo_index.outpoints()); + let mut wallet_coins = HashMap::new(); + // Go through all the wallet txos and create a coin for each. + for ((k, i), full_txo) in wallet_txos { + let outpoint = full_txo.outpoint; + if outpoints.map(|ops| !ops.contains(&outpoint)) == Some(true) { + continue; + } + let amount = full_txo.txout.value; + let derivation_index = i.into(); + let is_change = matches!(k, KeychainType::Change); + let block_info = match full_txo.chain_position { + ChainPosition::Unconfirmed(ls) => { + if let Some(last_seen) = last_seen.filter(|last_seen| *last_seen != ls) { + log::debug!("Ignoring coin at {}, which was last seen at {} instead of {} as required.", outpoint, ls, last_seen); + continue; + } + None + } + ChainPosition::Confirmed(anchor) => Some(block_info_from_anchor(anchor)), + }; + + // Immature if from a coinbase transaction with less than a hundred confs. + let is_immature = full_txo.is_on_coinbase + && block_info + .and_then(|blk| { + let tip_height: i32 = height_i32_from_u32(tip_id.height); + tip_height + .checked_sub(blk.height) + .map(|confs| confs < COINBASE_MATURITY) + }) + .unwrap_or(true); + + // Get spend status of this coin. + let (mut spend_txid, mut spend_block) = (None, None); + if let Some((spend_pos, txid)) = full_txo.spent_by { + spend_txid = Some(txid); + match spend_pos { + ChainPosition::Unconfirmed(ls) => { + if let Some(last_seen) = last_seen.filter(|last_seen| *last_seen != ls) { + log::debug!( + "Ignoring spend txid {} for coin at {}, \ + which was last seen at {} instead of {} as required.", + txid, + outpoint, + ls, + last_seen + ); + spend_txid = None; + } + } + ChainPosition::Confirmed(anchor) => { + spend_block = Some(block_info_from_anchor(anchor)); + } + }; + } + let coin = crate::bitcoin::Coin { + outpoint, + amount, + derivation_index, + is_change, + is_immature, + block_info, + spend_txid, + spend_block, + }; + wallet_coins.insert(coin.outpoint, coin); + } + wallet_coins + } + + pub fn get_transaction( + &self, + txid: &bitcoin::Txid, + ) -> Option<(bitcoin::Transaction, Option)> { + self.graph.graph().get_tx_node(*txid).map(|tx_node| { + let block = tx_node.anchors.iter().next().map(|info| Block { + hash: info.anchor_block.hash, // not necessarily the confirmation block hash + height: height_i32_from_u32(info.confirmation_height), + time: info.confirmation_time.try_into().expect("u32 by consensus"), + }); + let tx = tx_node.tx.as_ref().clone(); + (tx, block) + }) + } + + /// Find the first block in the local chain whose height is less than or equal to this. + pub fn find_block_at_or_before_height(&self, height: u32) -> BlockChainTip { + for cp in self.local_chain.iter_checkpoints() { + if cp.height() <= height { + return BlockChainTip { + height: height_i32_from_u32(cp.height()), + hash: cp.hash(), + }; + } + } + unreachable!("There must be at least the genesis block.") + } + + /// Apply an update to the local chain. + /// Panics if update does not connect to the local chain. + pub fn apply_connected_chain_update(&mut self, chain_update: CheckPoint) -> ChainChangeSet { + self.local_chain + .apply_update(chain_update) + .expect("update must connect to local chain") + } + + /// Apply a graph update. + pub fn apply_graph_update(&mut self, graph_update: TxGraph) { + let _ = self.graph.apply_update(graph_update); + } + + /// Apply a keychain update. + pub fn apply_keychain_update(&mut self, keychain_update: BTreeMap) { + let _ = self.graph.index.reveal_to_target_multi(&keychain_update); + } +} diff --git a/src/bitcoin/mod.rs b/src/bitcoin/mod.rs index a278d33aa..104d80d6f 100644 --- a/src/bitcoin/mod.rs +++ b/src/bitcoin/mod.rs @@ -3,17 +3,21 @@ //! Broadcast transactions, poll for new unspent coins, gather fee estimates. pub mod d; +pub mod electrum; pub mod poller; use crate::{ bitcoin::d::{BitcoindError, CachedTxGetter, LSBlockEntry}, descriptors, }; -pub use d::{MempoolEntry, SyncProgress}; +pub use d::{MempoolEntry, MempoolEntryFees, SyncProgress}; use std::{fmt, sync}; -use miniscript::bitcoin::{self, address}; +use miniscript::bitcoin::{self, address, bip32::ChildNumber}; + +// A spent coin's outpoint together with its spend transaction's txid, height and time. +type SpentCoin = (bitcoin::OutPoint, bitcoin::Txid, i32, u32); const COINBASE_MATURITY: i32 = 100; @@ -58,6 +62,17 @@ pub trait BitcoinInterface: Send { /// Check whether this former tip is part of the current best chain. fn is_in_chain(&self, tip: &BlockChainTip) -> bool; + /// Sync the wallet with the current best chain. + /// `receive_index` and `change_index` are the last derivation indices + /// that are expected to have been used by the wallet. + /// In case there has been a reorg, returns the common ancestor between + /// the wallet and the reorged chain. + fn sync_wallet( + &mut self, + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Result, String>; + /// Get coins received since the specified tip. fn received_coins( &self, @@ -84,10 +99,7 @@ pub trait BitcoinInterface: Send { fn spent_coins( &self, outpoints: &[(bitcoin::OutPoint, bitcoin::Txid)], - ) -> ( - Vec<(bitcoin::OutPoint, bitcoin::Txid, Block)>, - Vec, - ); + ) -> (Vec, Vec); /// Get the common ancestor between the Bitcoin backend's tip and the given tip. fn common_ancestor(&self, tip: &BlockChainTip) -> Option; @@ -98,7 +110,7 @@ pub trait BitcoinInterface: Send { /// Trigger a rescan of the block chain for transactions related to this descriptor since /// the given date. fn start_rescan( - &self, + &mut self, desc: &descriptors::LianaDescriptor, timestamp: u32, ) -> Result<(), String>; @@ -157,6 +169,15 @@ impl BitcoinInterface for d::BitcoinD { .unwrap_or(false) } + // The watchonly wallet handles this for us. + fn sync_wallet( + &mut self, + _receive_index: ChildNumber, + _change_index: ChildNumber, + ) -> Result, String> { + Ok(None) + } + fn received_coins( &self, tip: &BlockChainTip, @@ -184,7 +205,7 @@ impl BitcoinInterface for d::BitcoinD { outpoint, amount, block_height, - address, + address: UTxOAddress::Address(address), is_immature, }) } else { @@ -261,10 +282,7 @@ impl BitcoinInterface for d::BitcoinD { fn spent_coins( &self, outpoints: &[(bitcoin::OutPoint, bitcoin::Txid)], - ) -> ( - Vec<(bitcoin::OutPoint, bitcoin::Txid, Block)>, - Vec, - ) { + ) -> (Vec, Vec) { // Spend coins to be returned. let mut spent = Vec::with_capacity(outpoints.len()); // Coins whose spending transaction isn't in our local mempool anymore. @@ -282,7 +300,7 @@ impl BitcoinInterface for d::BitcoinD { // If the transaction was confirmed, mark it as such. if let Some(block) = res.block { - spent.push((*op, *txid, block)); + spent.push((*op, *txid, block.height, block.time)); continue; } @@ -305,7 +323,7 @@ impl BitcoinInterface for d::BitcoinD { }) }); if let Some((txid, block)) = conflict { - spent.push((*op, txid, block)); + spent.push((*op, txid, block.height, block.time)); continue; } @@ -347,7 +365,7 @@ impl BitcoinInterface for d::BitcoinD { } fn start_rescan( - &self, + &mut self, desc: &descriptors::LianaDescriptor, timestamp: u32, ) -> Result<(), String> { @@ -388,6 +406,193 @@ impl BitcoinInterface for d::BitcoinD { } } +impl BitcoinInterface for electrum::Electrum { + fn sync_wallet( + &mut self, + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Result, String> { + self.sync_wallet(receive_index, change_index) + .map_err(|e| e.to_string()) + } + + fn received_coins( + &self, + tip: &BlockChainTip, + _descs: &[descriptors::SinglePathLianaDesc], + ) -> Vec { + // Get those wallet coins that are either unconfirmed or have a confirmation height + // after tip. The poller will then discard any that had already been received. + self.wallet_coins(None) + .values() + .filter_map(|c| { + let height = c.block_info.map(|info| info.height); + if height.filter(|h| *h <= tip.height).is_some() { + None + } else { + Some(UTxO { + outpoint: c.outpoint, + block_height: height, + amount: c.amount, + address: UTxOAddress::DerivIndex(c.derivation_index, c.is_change), + is_immature: c.is_immature, + }) + } + }) + .collect() + } + + fn confirmed_coins( + &self, + outpoints: &[bitcoin::OutPoint], + ) -> (Vec<(bitcoin::OutPoint, i32, u32)>, Vec) { + let wallet_coins = &self.wallet_coins(Some(outpoints)); + let mut confirmed = Vec::new(); + let mut expired = Vec::new(); + for op in outpoints { + if let Some(w_c) = wallet_coins.get(op) { + if let Some(block) = w_c.block_info { + if w_c.is_immature { + log::debug!( + "Coin at '{}' comes from an immature coinbase transaction at \ + block height {}. Not marking it as confirmed for now.", + op, + block.height + ); + continue; + } + confirmed.push((w_c.outpoint, block.height, block.time)); + } + } else { + expired.push(*op); + } + } + (confirmed, expired) + } + + fn spending_coins( + &self, + outpoints: &[bitcoin::OutPoint], + ) -> Vec<(bitcoin::OutPoint, bitcoin::Txid)> { + let wallet_coins = &self.wallet_coins(Some(outpoints)); + outpoints + .iter() + .filter_map(|op| { + if let Some(w_c) = wallet_coins.get(op) { + w_c.spend_txid.map(|txid| (w_c.outpoint, txid)) + } else { + None + } + }) + .collect() + } + + fn spent_coins( + &self, + outpoints: &[(bitcoin::OutPoint, bitcoin::Txid)], + ) -> (Vec, Vec) { + let ops: Vec<_> = outpoints.iter().map(|(op, _)| op).copied().collect(); + let wallet_coins = &self.wallet_coins(Some(&ops)); + let mut spent = Vec::new(); + let mut expired_spending = Vec::new(); + + for (op, spend_txid) in outpoints { + if let Some(w_c) = wallet_coins.get(op) { + if w_c.spend_txid != Some(*spend_txid) { + expired_spending.push(*op); + } + if let Some(block) = w_c.spend_block { + spent.push((*op, *spend_txid, block.height, block.time)); + } + } + } + (spent, expired_spending) + } + + fn genesis_block_timestamp(&self) -> u32 { + self.client() + .genesis_block_timestamp() + .expect("Genesis block timestamp must always be there") + } + + fn genesis_block(&self) -> BlockChainTip { + self.client() + .genesis_block() + .expect("Genesis block must always be there") + } + + fn chain_tip(&self) -> BlockChainTip { + // We want the wallet's local chain tip after syncing. + self.wallet_tip() + } + + fn is_in_chain(&self, tip: &BlockChainTip) -> bool { + // Return `false` if no block at same height as `tip` + // is in wallet's local chain. + self.is_in_wallet_chain(*tip).unwrap_or_default() + } + + /// FIXME: make the Bitcoin backend interface higher level. See the comment in the poller next + /// to the `sync_wallet()` call. + fn common_ancestor(&self, _tip: &BlockChainTip) -> Option { + unreachable!("The common ancestor is returned in `sync_wallet()`. If no reorg was detected then, this method will never be called on an Electrum backend.") + } + + fn broadcast_tx(&self, tx: &bitcoin::Transaction) -> Result<(), String> { + match self.client().broadcast_tx(tx) { + Ok(_txid) => Ok(()), + Err(e) => Err(e.to_string()), + } + } + + fn wallet_transaction( + &self, + txid: &bitcoin::Txid, + ) -> Option<(bitcoin::Transaction, Option)> { + self.wallet_transaction(txid) + } + + fn mempool_entry(&self, txid: &bitcoin::Txid) -> Option { + self.client().mempool_entry(txid).ok()? + } + + fn mempool_spenders(&self, outpoints: &[bitcoin::OutPoint]) -> Vec { + self.client() + .mempool_spenders(outpoints) + .unwrap_or_default() + } + + fn sync_progress(&self) -> SyncProgress { + // Always return 100% for now since the API is bitcoind-specific to mean "blocks/headers". + // But in the future it would be nice to inform the user about the progress of the sync + // if it takes a few dozen seconds. + let blocks = self.chain_tip().height as u64; + SyncProgress::new(1.0, blocks, blocks) + } + + fn start_rescan( + &mut self, + _desc: &descriptors::LianaDescriptor, + _timestamp: u32, + ) -> Result<(), String> { + self.trigger_rescan(); + Ok(()) + } + + fn rescan_progress(&self) -> Option { + // Until we sync we're at 0%. After the sync, we're at 100%. + self.is_rescanning().then_some(0.0) + } + + fn block_before_date(&self, _timestamp: u32) -> Option { + Some(self.genesis_block()) + } + + fn tip_time(&self) -> Option { + self.client().tip_time().ok() + } +} + // FIXME: do we need to repeat the entire trait implemenation? Isn't there a nicer way? impl BitcoinInterface for sync::Arc> { fn genesis_block_timestamp(&self) -> u32 { @@ -410,6 +615,16 @@ impl BitcoinInterface for sync::Arc> self.lock().unwrap().is_in_chain(tip) } + fn sync_wallet( + &mut self, + receive_index: ChildNumber, + change_index: ChildNumber, + ) -> Result, String> { + self.lock() + .unwrap() + .sync_wallet(receive_index, change_index) + } + fn received_coins( &self, tip: &BlockChainTip, @@ -435,10 +650,7 @@ impl BitcoinInterface for sync::Arc> fn spent_coins( &self, outpoints: &[(bitcoin::OutPoint, bitcoin::Txid)], - ) -> ( - Vec<(bitcoin::OutPoint, bitcoin::Txid, Block)>, - Vec, - ) { + ) -> (Vec, Vec) { self.lock().unwrap().spent_coins(outpoints) } @@ -451,7 +663,7 @@ impl BitcoinInterface for sync::Arc> } fn start_rescan( - &self, + &mut self, desc: &descriptors::LianaDescriptor, timestamp: u32, ) -> Result<(), String> { @@ -493,6 +705,32 @@ pub struct UTxO { pub outpoint: bitcoin::OutPoint, pub amount: bitcoin::Amount, pub block_height: Option, - pub address: bitcoin::Address, + pub address: UTxOAddress, + pub is_immature: bool, +} + +/// Details about the UTXO address. +#[derive(Debug, Clone)] +pub enum UTxOAddress { + Address(bitcoin::Address), + /// Derivation index and whether it is from the change descriptor. + DerivIndex(ChildNumber, bool), +} + +#[derive(Debug, Clone, Copy)] +pub struct BlockInfo { + pub height: i32, + pub time: u32, +} + +#[derive(Debug, Clone, Copy)] +pub struct Coin { + pub outpoint: bitcoin::OutPoint, + pub amount: bitcoin::Amount, + pub derivation_index: ChildNumber, + pub is_change: bool, pub is_immature: bool, + pub block_info: Option, + pub spend_txid: Option, + pub spend_block: Option, } diff --git a/src/bitcoin/poller/looper.rs b/src/bitcoin/poller/looper.rs index 8ca1c73ea..20cc1d1b6 100644 --- a/src/bitcoin/poller/looper.rs +++ b/src/bitcoin/poller/looper.rs @@ -1,10 +1,10 @@ use crate::{ - bitcoin::{BitcoinInterface, BlockChainTip, UTxO}, + bitcoin::{BitcoinInterface, BlockChainTip, UTxO, UTxOAddress}, database::{Coin, DatabaseConnection, DatabaseInterface}, descriptors, }; -use std::{collections::HashSet, sync, time}; +use std::{collections::HashSet, sync, thread, time}; use miniscript::bitcoin::{self, secp256k1}; @@ -46,44 +46,53 @@ fn update_coins( .. } = utxo; // We can only really treat them if we know the derivation index that was used. - let address = match address.require_network(network) { - Ok(addr) => addr, - Err(e) => { - log::error!("Invalid network for address: {}", e); - continue; + let (derivation_index, is_change) = match address { + UTxOAddress::Address(address) => { + let address = match address.require_network(network) { + Ok(addr) => addr, + Err(e) => { + log::error!("Invalid network for address: {}", e); + continue; + } + }; + if let Some((derivation_index, is_change)) = + db_conn.derivation_index_by_address(&address) + { + (derivation_index, is_change) + } else { + // TODO: maybe we could try out something here? Like bruteforcing the next 200 indexes? + log::error!( + "Could not get derivation index for coin '{}' (address: '{}')", + &utxo.outpoint, + &address + ); + continue; + } } + UTxOAddress::DerivIndex(index, is_change) => (index, is_change), }; - if let Some((derivation_index, is_change)) = db_conn.derivation_index_by_address(&address) { - // First of if we are receiving coins that are beyond our next derivation index, - // adjust it. - if derivation_index > db_conn.receive_index() { - db_conn.set_receive_index(derivation_index, secp); - } - if derivation_index > db_conn.change_index() { - db_conn.set_change_index(derivation_index, secp); - } + // First of if we are receiving coins that are beyond our next derivation index, + // adjust it. + if derivation_index > db_conn.receive_index() { + db_conn.set_receive_index(derivation_index, secp); + } + if derivation_index > db_conn.change_index() { + db_conn.set_change_index(derivation_index, secp); + } - // Now record this coin as a newly received one. - if !curr_coins.contains_key(&utxo.outpoint) { - let coin = Coin { - outpoint, - is_immature, - amount, - derivation_index, - is_change, - block_info: None, - spend_txid: None, - spend_block: None, - }; - received.push(coin); - } - } else { - // TODO: maybe we could try out something here? Like bruteforcing the next 200 indexes? - log::error!( - "Could not get derivation index for coin '{}' (address: '{}')", - &utxo.outpoint, - &address - ); + // Now record this coin as a newly received one. + if !curr_coins.contains_key(&utxo.outpoint) { + let coin = Coin { + outpoint, + is_immature, + amount, + derivation_index, + is_change, + block_info: None, + spend_txid: None, + spend_block: None, + }; + received.push(coin); } } log::debug!("Newly received coins: {:?}", received); @@ -140,10 +149,6 @@ fn update_coins( .chain(spending.iter().cloned()) .collect(); let (spent, expired_spending) = bit.spent_coins(spending_coins.as_slice()); - let spent = spent - .into_iter() - .map(|(oupoint, txid, block)| (oupoint, txid, block.height, block.time)) - .collect(); log::debug!("Newly spent coins: {:?}", spent); UpdatedCoins { @@ -239,20 +244,44 @@ fn new_tip(bit: &impl BitcoinInterface, current_tip: &BlockChainTip) -> TipUpdat fn updates( db_conn: &mut Box, - bit: &impl BitcoinInterface, + bit: &mut impl BitcoinInterface, descs: &[descriptors::SinglePathLianaDesc], secp: &secp256k1::Secp256k1, ) { - // Check if there was a new block before updating ourselves. + // Check if there was a new block before we update our state. + // + // Some backends (such as Electrum) need to perform an explicit sync to provide updated data + // about the Bitcoin network. For those the common ancestor is immediately returned in case + // there was a reorg. For other backends (such as bitcoind) this function always return + // `Ok(None)`. We leverage this to query the next tip and poll for reorgs only in this case. + // FIXME: harmonize the Bitcoin backend interface, this intricacy is due to the introduction of + // an Electrum backend with the bitcoind-specific backend interface. let current_tip = db_conn.chain_tip().expect("Always set at first startup"); - let latest_tip = match new_tip(bit, ¤t_tip) { - TipUpdate::Same => current_tip, - TipUpdate::Progress(new_tip) => new_tip, - TipUpdate::Reorged(new_tip) => { + let (receive_index, change_index) = (db_conn.receive_index(), db_conn.change_index()); + let latest_tip = match bit.sync_wallet(receive_index, change_index) { + Ok(None) => { + match new_tip(bit, ¤t_tip) { + TipUpdate::Same => current_tip, + TipUpdate::Progress(new_tip) => new_tip, + TipUpdate::Reorged(new_tip) => { + // The block chain was reorganized. Rollback our state down to the common ancestor + // between our former chain and the new one, then restart fresh. + db_conn.rollback_tip(&new_tip); + log::info!("Tip was rolled back to '{}'.", new_tip); + return updates(db_conn, bit, descs, secp); + } + } + } + Ok(Some(reorg_common_ancestor)) => { // The block chain was reorganized. Rollback our state down to the common ancestor // between our former chain and the new one, then restart fresh. - db_conn.rollback_tip(&new_tip); - log::info!("Tip was rolled back to '{}'.", new_tip); + db_conn.rollback_tip(&reorg_common_ancestor); + log::info!("Tip was rolled back to '{}'.", &reorg_common_ancestor); + return updates(db_conn, bit, descs, secp); + } + Err(e) => { + log::error!("Error syncing wallet: '{}'.", e); + thread::sleep(time::Duration::from_secs(2)); return updates(db_conn, bit, descs, secp); } }; @@ -289,7 +318,7 @@ fn updates( // Check if there is any rescan of the backend ongoing or one that just finished. fn rescan_check( db_conn: &mut Box, - bit: &impl BitcoinInterface, + bit: &mut impl BitcoinInterface, descs: &[descriptors::SinglePathLianaDesc], secp: &secp256k1::Secp256k1, ) { @@ -356,7 +385,7 @@ pub fn sync_poll_interval() -> time::Duration { /// Update our state from the Bitcoin backend. pub fn poll( - bit: &sync::Arc>, + bit: &mut sync::Arc>, db: &sync::Arc>, secp: &secp256k1::Secp256k1, descs: &[descriptors::SinglePathLianaDesc], diff --git a/src/bitcoin/poller/mod.rs b/src/bitcoin/poller/mod.rs index 8e9874e16..c2986238e 100644 --- a/src/bitcoin/poller/mod.rs +++ b/src/bitcoin/poller/mod.rs @@ -56,7 +56,7 @@ impl Poller { /// Typically this would run for the whole duration of the program in a thread, and the main /// thread would set the `shutdown` atomic to `true` when shutting down. pub fn poll_forever( - &self, + &mut self, poll_interval: time::Duration, receiver: mpsc::Receiver, ) { @@ -91,7 +91,7 @@ impl Poller { // We've been asked to poll, don't wait any further and signal completion to // the caller. last_poll = Some(time::Instant::now()); - looper::poll(&self.bit, &self.db, &self.secp, &self.descs); + looper::poll(&mut self.bit, &self.db, &self.secp, &self.descs); if let Err(e) = sender.send(()) { log::error!("Error sending immediate poll completion signal: {}.", e); } @@ -122,7 +122,7 @@ impl Poller { } } - looper::poll(&self.bit, &self.db, &self.secp, &self.descs); + looper::poll(&mut self.bit, &self.db, &self.secp, &self.descs); } } } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 6e2b20636..092e87e74 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -986,7 +986,7 @@ impl DaemonControl { /// Trigger a rescan of the block chain for transactions involving our main descriptor between /// the given date and the current tip. /// The date must be after the genesis block time and before the current tip blocktime. - pub fn start_rescan(&self, timestamp: u32) -> Result<(), CommandError> { + pub fn start_rescan(&mut self, timestamp: u32) -> Result<(), CommandError> { let mut db_conn = self.db.connection(); let genesis_timestamp = self.bitcoin.genesis_block_timestamp(); diff --git a/src/config.rs b/src/config.rs index c0f24ac12..d82cf32bd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -85,6 +85,17 @@ fn default_daemon() -> bool { false } +/// Bitcoin backend config. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub enum BitcoinBackend { + /// Settings specific to bitcoind as the Bitcoin interface. + #[serde(rename = "bitcoind_config")] + Bitcoind(BitcoindConfig), + /// Settings specific to Electrum as the Bitcoin interface. + #[serde(rename = "electrum_config")] + Electrum(ElectrumConfig), +} + /// RPC authentication options. #[derive(Clone, PartialEq, Serialize)] pub enum BitcoindRpcAuth { @@ -115,6 +126,15 @@ pub struct BitcoindConfig { pub addr: SocketAddr, } +/// Everything we need to know for talking to Electrum serenely. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ElectrumConfig { + /// The URL the Electrum's RPC is listening on. + /// Include "ssl://" for SSL. otherwise TCP will be assumed. + /// Can optionally prefix with "tcp://". + pub addr: String, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct BitcoinConfig { /// The network we are operating on, one of "bitcoin", "testnet", "regtest", "signet" @@ -152,8 +172,9 @@ pub struct Config { pub main_descriptor: LianaDescriptor, /// Settings for the Bitcoin interface pub bitcoin_config: BitcoinConfig, - /// Settings specific to bitcoind as the Bitcoin interface - pub bitcoind_config: Option, + /// Settings specific to the Bitcoin backend. + #[serde(flatten)] + pub bitcoin_backend: Option, } impl Config { diff --git a/src/descriptors/mod.rs b/src/descriptors/mod.rs index 7ff6ac30d..818ecb19c 100644 --- a/src/descriptors/mod.rs +++ b/src/descriptors/mod.rs @@ -613,6 +613,13 @@ impl SinglePathLianaDesc { ), ) } + + /// Reference to the underlying `Descriptor` + pub fn as_descriptor_public_key( + &self, + ) -> &descriptor::Descriptor { + &self.0 + } } pub enum DescKeysOrigins { diff --git a/src/jsonrpc/api.rs b/src/jsonrpc/api.rs index 5835e2e9c..3ff137afb 100644 --- a/src/jsonrpc/api.rs +++ b/src/jsonrpc/api.rs @@ -266,7 +266,7 @@ fn list_transactions(control: &DaemonControl, params: Params) -> Result Result { +fn start_rescan(control: &mut DaemonControl, params: Params) -> Result { let timestamp: u32 = params .get(0, "timestamp") .ok_or_else(|| Error::invalid_params("Missing 'timestamp' parameter."))? @@ -365,7 +365,7 @@ fn get_labels(control: &DaemonControl, params: Params) -> Result Result { +pub fn handle_request(control: &mut DaemonControl, req: Request) -> Result { let result = match req.method.as_str() { "broadcastspend" => { let params = req diff --git a/src/jsonrpc/server.rs b/src/jsonrpc/server.rs index 424adc2a0..49b0d925a 100644 --- a/src/jsonrpc/server.rs +++ b/src/jsonrpc/server.rs @@ -81,7 +81,7 @@ fn read_command( // Handle all messages from this connection. fn connection_handler( - control: DaemonControl, + mut control: DaemonControl, mut stream: net::UnixStream, shutdown: sync::Arc, ) -> Result<(), io::Error> { @@ -106,7 +106,7 @@ fn connection_handler( log::trace!("JSONRPC request: {:?}", serde_json::to_string(&req)); let response = - api::handle_request(&control, req).unwrap_or_else(|e| Response::error(req_id, e)); + api::handle_request(&mut control, req).unwrap_or_else(|e| Response::error(req_id, e)); log::trace!("JSONRPC response: {:?}", serde_json::to_string(&response)); if let Err(e) = serde_json::to_writer(&stream, &response) { log::error!("Error writing response: '{}'", e); diff --git a/src/lib.rs b/src/lib.rs index 584a3f3c7..b43ca31b2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,10 +13,15 @@ pub mod spend; #[cfg(test)] mod testutils; +pub use bdk_electrum::electrum_client; pub use bip39; +use bitcoin::electrum; pub use miniscript; -pub use crate::bitcoin::d::{BitcoinD, BitcoindError, WalletError}; +pub use crate::bitcoin::{ + d::{BitcoinD, BitcoindError, WalletError}, + electrum::{Electrum, ElectrumError}, +}; #[cfg(feature = "daemon")] use crate::jsonrpc::server::{rpcserver_loop, rpcserver_setup}; use crate::{ @@ -34,7 +39,7 @@ use std::{ thread, }; -use miniscript::bitcoin::secp256k1; +use miniscript::bitcoin::{constants::ChainHash, hashes::Hash, secp256k1, BlockHash}; #[cfg(not(test))] use std::panic; @@ -92,9 +97,12 @@ pub enum StartupError { DefaultDataDirNotFound, DatadirCreation(path::PathBuf, io::Error), MissingBitcoindConfig, + MissingElectrumConfig, + MissingBitcoinBackendConfig, DbMigrateBitcoinTxs(&'static str), Database(SqliteDbError), Bitcoind(BitcoindError), + Electrum(ElectrumError), #[cfg(unix)] Daemonization(&'static str), #[cfg(windows)] @@ -117,12 +125,21 @@ impl fmt::Display for StartupError { f, "Our Bitcoin interface is bitcoind but we have no 'bitcoind_config' entry in the configuration." ), + Self::MissingElectrumConfig => write!( + f, + "Our Bitcoin interface is Electrum but we have no 'electrum_config' entry in the configuration." + ), + Self::MissingBitcoinBackendConfig => write!( + f, + "No Bitcoin backend entry in the configuration." + ), Self::DbMigrateBitcoinTxs(msg) => write!( f, "Error when migrating Bitcoin transaction from Bitcoin backend to database: {}.", msg ), Self::Database(e) => write!(f, "Error initializing database: '{}'.", e), Self::Bitcoind(e) => write!(f, "Error setting up bitcoind interface: '{}'.", e), + Self::Electrum(e) => write!(f, "Error setting up Electrum interface: '{}'.", e), #[cfg(unix)] Self::Daemonization(e) => write!(f, "Error when daemonizing: '{}'.", e), #[cfg(windows)] @@ -260,10 +277,10 @@ fn setup_bitcoind( #[cfg(target_os = "windows")] let wo_path_str = wo_path_str.replace("\\\\?\\", "").replace("\\\\?", ""); - let bitcoind_config = config - .bitcoind_config - .as_ref() - .ok_or(StartupError::MissingBitcoindConfig)?; + let bitcoind_config = match config.bitcoin_backend.as_ref() { + Some(config::BitcoinBackend::Bitcoind(bitcoind_config)) => bitcoind_config, + _ => Err(StartupError::MissingBitcoindConfig)?, + }; let bitcoind = BitcoinD::new(bitcoind_config, wo_path_str)?; bitcoind.node_sanity_checks( config.bitcoin_config.network, @@ -287,6 +304,70 @@ fn setup_bitcoind( Ok(bitcoind) } +// Create an Electrum interface from a client and BDK-based wallet, and do some sanity checks. +// If all went well, returns the interface to Electrum. +fn setup_electrum( + config: &Config, + db: sync::Arc>, +) -> Result { + let electrum_config = match config.bitcoin_backend.as_ref() { + Some(config::BitcoinBackend::Electrum(electrum_config)) => electrum_config, + _ => Err(StartupError::MissingElectrumConfig)?, + }; + // First create the client to communicate with the Electrum server. + let client = electrum::client::Client::new(electrum_config) + .map_err(|e| StartupError::Electrum(ElectrumError::Client(e)))?; + // Then create the BDK-based wallet and populate it with DB data. + let mut db_conn = db.connection(); + let tip = db_conn.chain_tip(); + let coins: Vec<_> = db_conn + .coins(&[], &[]) + .into_values() + .map(|c| crate::bitcoin::Coin { + outpoint: c.outpoint, + amount: c.amount, + derivation_index: c.derivation_index, + is_change: c.is_change, + is_immature: c.is_immature, + block_info: c.block_info.map(|info| crate::bitcoin::BlockInfo { + height: info.height, + time: info.time, + }), + spend_txid: c.spend_txid, + spend_block: c.spend_block.map(|info| crate::bitcoin::BlockInfo { + height: info.height, + time: info.time, + }), + }) + .collect(); + let txids = db_conn.list_saved_txids(); + // This will only return those txs referenced by our coins, which may not be all of `txids`. + let txs: Vec<_> = db_conn + .list_wallet_transactions(&txids) + .into_iter() + .map(|(tx, _, _)| tx) + .collect(); + let (receive_index, change_index) = (db_conn.receive_index(), db_conn.change_index()); + let genesis_hash = { + let chain_hash = ChainHash::using_genesis_block(config.bitcoin_config.network); + BlockHash::from_byte_array(*chain_hash.as_bytes()) + }; + let bdk_wallet = electrum::wallet::BdkWallet::new( + &config.main_descriptor, + genesis_hash, + tip, + &coins, + &txs, + receive_index, + change_index, + ); + let electrum = Electrum::new(client, bdk_wallet).map_err(StartupError::Electrum)?; + electrum + .sanity_checks(&genesis_hash) + .map_err(StartupError::Electrum)?; + Ok(electrum) +} + #[derive(Clone)] pub struct DaemonControl { config: Config, @@ -374,7 +455,11 @@ impl DaemonHandle { // Set up the connection to bitcoind (if using it) first as we may need it for the database // migration when setting up SQLite below. let bitcoind = if bitcoin.is_none() { - Some(setup_bitcoind(&config, &data_dir, fresh_data_dir)?) + if let Some(config::BitcoinBackend::Bitcoind(_)) = &config.bitcoin_backend { + Some(setup_bitcoind(&config, &data_dir, fresh_data_dir)?) + } else { + None + } } else { None }; @@ -392,11 +477,16 @@ impl DaemonHandle { }; // Finally set up the Bitcoin backend. - let bit = match (bitcoin, bitcoind) { - (Some(bit), None) => sync::Arc::from(sync::Mutex::from(bit)), - (None, Some(bit)) => sync::Arc::from(sync::Mutex::from(bit)) + let bit = match (bitcoin, &config.bitcoin_backend) { + (Some(bit), _) => sync::Arc::from(sync::Mutex::from(bit)), + (None, Some(config::BitcoinBackend::Bitcoind(..))) => sync::Arc::from( + sync::Mutex::from(bitcoind.expect("bitcoind must have been set already")), + ) as sync::Arc>, - _ => unreachable!("Either bitcoind or bitcoin interface is always set."), + (None, Some(config::BitcoinBackend::Electrum(..))) => { + sync::Arc::from(sync::Mutex::from(setup_electrum(&config, db.clone())?)) + } + (None, None) => Err(StartupError::MissingBitcoinBackendConfig)?, }; // If we are on a UNIX system and they told us to daemonize, do it now. @@ -415,7 +505,7 @@ impl DaemonHandle { // Start the poller thread. Keep the thread handle to be able to check if it crashed. Store // an atomic to be able to stop it. - let bitcoin_poller = + let mut bitcoin_poller = poller::Poller::new(bit.clone(), db.clone(), config.main_descriptor.clone()); let (poller_sender, poller_receiver) = mpsc::sync_channel(0); let poller_handle = thread::Builder::new() @@ -764,7 +854,7 @@ mod tests { let change_desc = desc.change_descriptor().clone(); let config = Config { bitcoin_config, - bitcoind_config: Some(bitcoind_config), + bitcoin_backend: Some(config::BitcoinBackend::Bitcoind(bitcoind_config)), data_dir: Some(data_dir), #[cfg(unix)] daemon: false, diff --git a/src/testutils.rs b/src/testutils.rs index 3669b5fc0..a5642b70f 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -64,6 +64,14 @@ impl BitcoinInterface for DummyBitcoind { true } + fn sync_wallet( + &mut self, + _receive_index: bip32::ChildNumber, + _change_index: bip32::ChildNumber, + ) -> Result, String> { + Ok(None) + } + fn received_coins( &self, _: &BlockChainTip, @@ -87,7 +95,7 @@ impl BitcoinInterface for DummyBitcoind { &self, _: &[(bitcoin::OutPoint, bitcoin::Txid)], ) -> ( - Vec<(bitcoin::OutPoint, bitcoin::Txid, Block)>, + Vec<(bitcoin::OutPoint, bitcoin::Txid, i32, u32)>, Vec, ) { (Vec::new(), Vec::new()) @@ -101,7 +109,7 @@ impl BitcoinInterface for DummyBitcoind { todo!() } - fn start_rescan(&self, _: &descriptors::LianaDescriptor, _: u32) -> Result<(), String> { + fn start_rescan(&mut self, _: &descriptors::LianaDescriptor, _: u32) -> Result<(), String> { todo!() } @@ -534,7 +542,7 @@ impl DummyLiana { let desc = descriptors::LianaDescriptor::new(policy); let config = Config { bitcoin_config, - bitcoind_config: None, + bitcoin_backend: None, data_dir: Some(data_dir), #[cfg(unix)] daemon: false, diff --git a/tests/fixtures.py b/tests/fixtures.py index 68b6e9ee1..4c362c835 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -3,9 +3,15 @@ from bip380.descriptors import Descriptor from concurrent import futures from test_framework.bitcoind import Bitcoind +from test_framework.electrs import Electrs from test_framework.lianad import Lianad from test_framework.signer import SingleSigner, MultiSigner -from test_framework.utils import EXECUTOR_WORKERS, USE_TAPROOT +from test_framework.utils import ( + BITCOIN_BACKEND_TYPE, + EXECUTOR_WORKERS, + USE_TAPROOT, + BitcoinBackendType, +) import hashlib import os @@ -115,6 +121,26 @@ def bitcoind(directory): bitcoind.cleanup() +@pytest.fixture +def bitcoin_backend(directory, bitcoind): + + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + yield bitcoind + bitcoind.cleanup() + elif BITCOIN_BACKEND_TYPE is BitcoinBackendType.Electrs: + electrs = Electrs( + electrs_dir=os.path.join(directory, "electrs"), + bitcoind_dir=bitcoind.bitcoin_dir, + bitcoind_rpcport=bitcoind.rpcport, + bitcoind_p2pport=bitcoind.p2pport, + ) + electrs.startup() + yield electrs + electrs.cleanup() + else: + raise NotImplementedError + + def xpub_fingerprint(hd): return _pubkey_to_fingerprint(hd.pubkey).hex() @@ -127,10 +153,9 @@ def single_key_desc(prim_fg, prim_xpub, reco_fg, reco_xpub, csv_value, is_taproo @pytest.fixture -def lianad(bitcoind, directory): +def lianad(bitcoin_backend, directory): datadir = os.path.join(directory, "lianad") os.makedirs(datadir, exist_ok=True) - bitcoind_cookie = os.path.join(bitcoind.bitcoin_dir, "regtest", ".cookie") signer = SingleSigner(is_taproot=USE_TAPROOT) (prim_fingerprint, primary_xpub), (reco_fingerprint, recovery_xpub) = ( @@ -155,8 +180,7 @@ def lianad(bitcoind, directory): datadir, signer, main_desc, - bitcoind.rpcport, - bitcoind_cookie, + bitcoin_backend, ) try: @@ -208,10 +232,9 @@ def multisig_desc(multi_signer, csv_value, is_taproot): @pytest.fixture -def lianad_multisig(bitcoind, directory): +def lianad_multisig(bitcoin_backend, directory): datadir = os.path.join(directory, "lianad") os.makedirs(datadir, exist_ok=True) - bitcoind_cookie = os.path.join(bitcoind.bitcoin_dir, "regtest", ".cookie") # A 3-of-4 that degrades into a 2-of-5 after 10 blocks csv_value = 10 @@ -224,8 +247,7 @@ def lianad_multisig(bitcoind, directory): datadir, signer, main_desc, - bitcoind.rpcport, - bitcoind_cookie, + bitcoin_backend, ) try: @@ -261,10 +283,9 @@ def multipath_desc(multi_signer, csv_values, is_taproot): @pytest.fixture -def lianad_multipath(bitcoind, directory): +def lianad_multipath(bitcoin_backend, directory): datadir = os.path.join(directory, "lianad") os.makedirs(datadir, exist_ok=True) - bitcoind_cookie = os.path.join(bitcoind.bitcoin_dir, "regtest", ".cookie") # A 3-of-4 that degrades into a 3-of-5 after 10 blocks and into a 1-of-10 after 20 blocks. csv_values = [10, 20] @@ -279,8 +300,7 @@ def lianad_multipath(bitcoind, directory): datadir, signer, main_desc, - bitcoind.rpcport, - bitcoind_cookie, + bitcoin_backend, ) try: diff --git a/tests/test_chain.py b/tests/test_chain.py index 26487c999..b4bf5960c 100644 --- a/tests/test_chain.py +++ b/tests/test_chain.py @@ -238,7 +238,8 @@ def reorg_shift(height, txs): outpoints_before = set(c["outpoint"] for c in coins_before) bitcoind.generate_block(1) lianad.restart_fresh(bitcoind) - assert len(list_coins()) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(list_coins()) == 0 # We can be stopped while we are rescanning lianad.rpc.startrescan(initial_tip["time"]) @@ -252,7 +253,8 @@ def reorg_shift(height, txs): bitcoind.generate_block(1) lianad.restart_fresh(bitcoind) wait_for(lambda: lianad.rpc.getinfo()["rescan_progress"] is None) - assert len(list_coins()) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(list_coins()) == 0 # There can be a reorg when we start rescanning reorg_shift(initial_tip["height"], txs) @@ -271,7 +273,8 @@ def reorg_shift(height, txs): lianad.restart_fresh(bitcoind) wait_synced() wait_for(lambda: lianad.rpc.getinfo()["rescan_progress"] is None) - assert len(list_coins()) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(list_coins()) == 0 # We can be rescanning when a reorg happens lianad.rpc.startrescan(initial_tip["time"]) @@ -350,7 +353,8 @@ def test_rescan_and_recovery(lianad, bitcoind): # Clear lianad state lianad.restart_fresh(bitcoind) - assert len(lianad.rpc.listcoins()["coins"]) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(lianad.rpc.listcoins()["coins"]) == 0 # Start rescan lianad.rpc.startrescan(initial_tip["time"]) diff --git a/tests/test_framework/bitcoind.py b/tests/test_framework/bitcoind.py index ab8e69ac3..a8a49b7ee 100644 --- a/tests/test_framework/bitcoind.py +++ b/tests/test_framework/bitcoind.py @@ -7,7 +7,14 @@ from decimal import Decimal from ephemeral_port_reserve import reserve from test_framework.authproxy import AuthServiceProxy -from test_framework.utils import TailableProc, wait_for, TIMEOUT, BITCOIND_PATH, COIN +from test_framework.utils import ( + BitcoinBackend, + TailableProc, + wait_for, + TIMEOUT, + BITCOIND_PATH, + COIN, +) class BitcoindRpcInterface: @@ -35,7 +42,7 @@ def f(*args): return f -class Bitcoind(TailableProc): +class Bitcoind(BitcoinBackend): def __init__(self, bitcoin_dir, rpcport=None): TailableProc.__init__(self, bitcoin_dir, verbose=False) @@ -65,6 +72,12 @@ def __init__(self, bitcoin_dir, rpcport=None): "rpcport": rpcport, "fallbackfee": Decimal(1000) / COIN, "rpcthreads": 32, + # bitcoind uses mocktime in some tests, which can lead to peers (e.g. electrs) + # being disconnected. To prevent this, we set `peertimeout` greater than + # the max value being mocked. + # See https://github.com/bitcoin/bitcoin/blob/fa05ee0517d58b600f0ccad4c02c0734a23707d6/src/net.cpp#L1961. + # h/t pythcoiner :) + "peertimeout": 2 * 24 * 60 * 60, # 2 days } self.conf_file = os.path.join(bitcoin_dir, "bitcoin.conf") with open(self.conf_file, "w") as f: @@ -275,3 +288,10 @@ def cleanup(self): except Exception: self.proc.kill() self.proc.wait() + + def append_to_lianad_conf(self, conf_file): + cookie_path = os.path.join(self.bitcoin_dir, "regtest", ".cookie") + with open(conf_file, "a") as f: + f.write("[bitcoind_config]\n") + f.write(f"cookie_path = '{cookie_path}'\n") + f.write(f"addr = '127.0.0.1:{self.rpcport}'\n") diff --git a/tests/test_framework/electrs.py b/tests/test_framework/electrs.py new file mode 100644 index 000000000..ee8c6a518 --- /dev/null +++ b/tests/test_framework/electrs.py @@ -0,0 +1,79 @@ +import logging +import os + +from ephemeral_port_reserve import reserve +from test_framework.utils import BitcoinBackend, TailableProc, ELECTRS_PATH + + +class Electrs(BitcoinBackend): + def __init__( + self, + bitcoind_dir, + bitcoind_rpcport, + bitcoind_p2pport, + electrs_dir, + rpcport=None, + ): + TailableProc.__init__(self, electrs_dir, verbose=False) + + if rpcport is None: + rpcport = reserve() + + # Prometheus metrics can't be deactivated in Electrs. Configure the port so it doesn't + # conflict with other instances when running tests in parallel. + monitoring_port = reserve() + + self.electrs_dir = electrs_dir + self.rpcport = rpcport + + regtestdir = os.path.join(electrs_dir, "regtest") + if not os.path.exists(regtestdir): + os.makedirs(regtestdir) + + self.cmd_line = [ + ELECTRS_PATH, + "--conf", + "{}/electrs.toml".format(regtestdir), + ] + electrs_conf = { + "daemon_dir": bitcoind_dir, + "cookie_file": os.path.join(bitcoind_dir, "regtest", ".cookie"), + "daemon_rpc_addr": f"127.0.0.1:{bitcoind_rpcport}", + "daemon_p2p_addr": f"127.0.0.1:{bitcoind_p2pport}", + "db_dir": electrs_dir, + "network": "regtest", + "electrum_rpc_addr": f"127.0.0.1:{self.rpcport}", + "monitoring_addr": f"127.0.0.1:{monitoring_port}", + } + self.conf_file = os.path.join(regtestdir, "electrs.toml") + with open(self.conf_file, "w") as f: + for k, v in electrs_conf.items(): + f.write(f'{k} = "{v}"\n') + + self.env = {"RUST_LOG": "DEBUG"} + + def start(self): + TailableProc.start(self) + logging.info("Electrs started") + + def startup(self): + try: + self.start() + except Exception: + self.stop() + raise + + def stop(self): + return TailableProc.stop(self) + + def cleanup(self): + try: + self.stop() + except Exception: + self.proc.kill() + self.proc.wait() + + def append_to_lianad_conf(self, conf_file): + with open(conf_file, "a") as f: + f.write("[electrum_config]\n") + f.write(f"addr = '127.0.0.1:{self.rpcport}'\n") diff --git a/tests/test_framework/lianad.py b/tests/test_framework/lianad.py index dfd81cd9d..5fcf9a38b 100644 --- a/tests/test_framework/lianad.py +++ b/tests/test_framework/lianad.py @@ -5,6 +5,8 @@ from bip380.descriptors import Descriptor from bip380.miniscript import SatisfactionMaterial from test_framework.utils import ( + BITCOIN_BACKEND_TYPE, + BitcoinBackendType, UnixDomainSocketRpc, TailableProc, VERBOSE, @@ -28,8 +30,7 @@ def __init__( datadir, signer, multi_desc, - bitcoind_rpc_port, - bitcoind_cookie_path, + bitcoin_backend, ): TailableProc.__init__(self, datadir, verbose=VERBOSE) @@ -44,6 +45,7 @@ def __init__( self.cmd_line = [LIANAD_PATH, "--conf", f"{self.conf_file}"] socket_path = os.path.join(os.path.join(datadir, "regtest"), "lianad_rpc") self.rpc = UnixDomainSocketRpc(socket_path) + self.bitcoin_backend = bitcoin_backend with open(self.conf_file, "w") as f: f.write(f"data_dir = '{datadir}'\n") @@ -55,10 +57,7 @@ def __init__( f.write("[bitcoin_config]\n") f.write('network = "regtest"\n') f.write("poll_interval_secs = 1\n") - - f.write("[bitcoind_config]\n") - f.write(f"cookie_path = '{bitcoind_cookie_path}'\n") - f.write(f"addr = '127.0.0.1:{bitcoind_rpc_port}'\n") + bitcoin_backend.append_to_lianad_conf(self.conf_file) def finalize_psbt(self, psbt): """Create a valid witness for all inputs in the PSBT. @@ -107,8 +106,9 @@ def restart_fresh(self, bitcoind): self.stop() dir_path = os.path.join(self.datadir, "regtest") shutil.rmtree(dir_path) - wallet_path = os.path.join(dir_path, "lianad_watchonly_wallet") - bitcoind.node_rpc.unloadwallet(wallet_path) + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + wallet_path = os.path.join(dir_path, "lianad_watchonly_wallet") + bitcoind.node_rpc.unloadwallet(wallet_path) self.start() wait_for( lambda: self.rpc.getinfo()["block_height"] == bitcoind.rpc.getblockcount() diff --git a/tests/test_framework/utils.py b/tests/test_framework/utils.py index 7f961038b..c48edc481 100644 --- a/tests/test_framework/utils.py +++ b/tests/test_framework/utils.py @@ -1,3 +1,5 @@ +import abc +import enum import itertools import json import logging @@ -20,8 +22,21 @@ os.path.dirname(__file__), "..", "..", "target/debug/lianad" ) LIANAD_PATH = os.getenv("LIANAD_PATH", DEFAULT_MS_PATH) + + +class BitcoinBackendType(str, enum.Enum): + Bitcoind = "bitcoind" + Electrs = "electrs" + + +DEFAULT_BITCOIN_BACKEND_TYPE = "bitcoind" +BITCOIN_BACKEND_TYPE = BitcoinBackendType( + os.getenv("BITCOIN_BACKEND_TYPE", DEFAULT_BITCOIN_BACKEND_TYPE) +) DEFAULT_BITCOIND_PATH = "bitcoind" BITCOIND_PATH = os.getenv("BITCOIND_PATH", DEFAULT_BITCOIND_PATH) +DEFAULT_ELECTRS_PATH = "electrs" +ELECTRS_PATH = os.getenv("ELECTRS_PATH", DEFAULT_ELECTRS_PATH) OLD_LIANAD_PATH = os.getenv("OLD_LIANAD_PATH", None) IS_NOT_BITCOIND_24 = bool(int(os.getenv("IS_NOT_BITCOIND_24", True))) USE_TAPROOT = bool( @@ -421,3 +436,12 @@ def wait_for_log(self, regex, timeout=TIMEOUT): Convenience wrapper for the common case of only seeking a single entry. """ return self.wait_for_logs([regex], timeout) + + +class BitcoinBackend(abc.ABC, TailableProc): + """All Bitcoin backends should derive from this class.""" + + @abc.abstractmethod + def append_to_lianad_conf(self, conf_file): + """Append backend config values to lianad config file.""" + ... diff --git a/tests/test_misc.py b/tests/test_misc.py index fecad7c3d..7cd1f95bb 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -7,6 +7,8 @@ from test_framework.authproxy import JSONRPCException from test_framework.serializations import PSBT from test_framework.utils import ( + BitcoinBackendType, + BITCOIN_BACKEND_TYPE, wait_for, RpcError, OLD_LIANAD_PATH, @@ -260,6 +262,10 @@ def test_coinbase_deposit(lianad, bitcoind): OLD_LIANAD_PATH is None or USE_TAPROOT, reason="Need the old lianad binary to create the datadir.", ) +@pytest.mark.skipif( + BITCOIN_BACKEND_TYPE is not BitcoinBackendType.Bitcoind, + reason="Only bitcoind backend was available for older lianad versions.", +) def test_migration(lianad_multisig, bitcoind): """Test we can start a newer lianad on a datadir created by an older lianad.""" lianad = lianad_multisig @@ -315,6 +321,10 @@ def bitcoind_wait_new_block(bitcoind): @pytest.mark.skipif( not IS_NOT_BITCOIND_24, reason="Need 'generateblock' with 'submit=False'" ) +@pytest.mark.skipif( + BITCOIN_BACKEND_TYPE is not BitcoinBackendType.Bitcoind, + reason="Tests the retry logic specific to the bitcoind backend.", +) def test_retry_on_workqueue_exceeded(lianad, bitcoind, executor): """Make sure we retry requests to bitcoind if it is temporarily overloaded.""" # Start by reducing the work queue to a single slot. Note we need to stop lianad diff --git a/tests/test_rpc.py b/tests/test_rpc.py index 675f8001f..9d9aef1ec 100644 --- a/tests/test_rpc.py +++ b/tests/test_rpc.py @@ -686,11 +686,13 @@ def all_spent(coins): # descriptor. coins_before = sorted_coins() lianad.restart_fresh(bitcoind) - assert len(list_coins()) == 0 + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert len(list_coins()) == 0 # The wallet isn't aware what derivation indexes were used. Necessarily it'll start # from 0. - assert lianad.rpc.getnewaddress() == first_address + if BITCOIN_BACKEND_TYPE is BitcoinBackendType.Bitcoind: + assert lianad.rpc.getnewaddress() == first_address # Once the rescan is done, we must have detected all previous transactions. lianad.rpc.startrescan(initial_timestamp) @@ -900,6 +902,9 @@ def test_create_recovery(lianad, bitcoind): bitcoind.generate_block(9, wait_for_mempool=txid) # Now we can create a recovery tx that sweeps the first 3 coins. + wait_for( + lambda: lianad.rpc.getinfo()["block_height"] == bitcoind.rpc.getblockcount() + ) res = lianad.rpc.createrecovery(bitcoind.rpc.getnewaddress(), 18) reco_psbt = PSBT.from_base64(res["psbt"]) @@ -1145,7 +1150,8 @@ def test_rbfpsbt_bump_fee(lianad, bitcoind): # feerate to set the min feerate, instead of 1 sat/vb of first # transaction: with pytest.raises( - RpcError, match=f"Feerate {int(rbf_1_feerate)} too low for minimum feerate 10." + RpcError, + match=f"Feerate {int(rbf_1_feerate)} too low for minimum feerate {int(rbf_1_feerate) + 1}.", ): lianad.rpc.rbfpsbt(first_txid, False, int(rbf_1_feerate)) # Using 1 more for feerate works.