diff --git a/Cargo.lock b/Cargo.lock index ec11587fa3..ca6200024e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,7 +505,6 @@ checksum = "9a41df6ad9642c5c15ae312dd3d074de38fd3eb7cc87ad4ce10f90292a83fe4d" dependencies = [ "bech32", "bitcoin_hashes", - "bitcoinconsensus", "secp256k1", ] @@ -515,16 +514,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "006cc91e1a1d99819bc5b8214be3555c1f0611b169f527a1fdc54ed1f2b745b0" -[[package]] -name = "bitcoinconsensus" -version = "0.19.0-3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a8aa43b5cd02f856cb126a9af819e77b8910fdd74dd1407be649f2f5fe3a1b5" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "bitcrypto" version = "0.1.0" @@ -2128,7 +2117,7 @@ dependencies = [ "impl-rlp", "impl-serde", "primitive-types", - "uint 0.9.1", + "uint 0.9.3", ] [[package]] @@ -3729,19 +3718,16 @@ version = "0.0.106" source = "git+https://github.com/shamardy/rust-lightning?branch=0.0.106#af4a89c08c22d0110d386df0e288b2f825aaebbc" dependencies = [ "bitcoin", - "hex 0.4.2", - "regex", "secp256k1", ] [[package]] name = "lightning-background-processor" version = "0.0.106" +source = "git+https://github.com/shamardy/rust-lightning?branch=0.0.106#af4a89c08c22d0110d386df0e288b2f825aaebbc" dependencies = [ "bitcoin", - "db_common", "lightning", - "lightning-invoice", "lightning-persister", ] @@ -3770,21 +3756,11 @@ dependencies = [ [[package]] name = "lightning-persister" version = "0.0.106" +source = "git+https://github.com/shamardy/rust-lightning?branch=0.0.106#af4a89c08c22d0110d386df0e288b2f825aaebbc" dependencies = [ - "async-trait", "bitcoin", - "common", - "db_common", - "derive_more", - "hex 0.4.2", "libc", "lightning", - "mm2_io", - "parking_lot 0.12.0", - "rand 0.7.3", - "secp256k1", - "serde", - "serde_json", "winapi", ] @@ -5014,16 +4990,17 @@ dependencies = [ "impl-rlp", "impl-serde", "scale-info", - "uint 0.9.1", + "uint 0.9.3", ] [[package]] name = "primitives" version = "0.1.0" dependencies = [ + "bitcoin_hashes", "byteorder 1.4.3", "rustc-hex 2.1.0", - "uint 0.9.1", + "uint 0.9.3", ] [[package]] @@ -7242,7 +7219,12 @@ dependencies = [ name = "spv_validation" version = "0.1.0" dependencies = [ + "async-trait", "chain", + "common", + "derive_more", + "keys", + "lazy_static", "primitives", "ripemd160", "rustc-hex 2.1.0", @@ -8148,9 +8130,9 @@ dependencies = [ [[package]] name = "uint" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6470ab50f482bde894a037a57064480a246dbfdd5960bd65a44824693f08da5f" +checksum = "12f03af7ccf01dd611cc450a0d10dbc9b745770d096473e2faf0ca6e2d66d1e0" dependencies = [ "byteorder 1.4.3", "crunchy 0.2.2", diff --git a/Cargo.toml b/Cargo.toml index 4e27bed80d..e224d8eafb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,6 @@ [workspace] members = [ "mm2src/coins", - "mm2src/coins/lightning_persister", - "mm2src/coins/lightning_background_processor", "mm2src/coins/utxo_signer", "mm2src/coins_activation", "mm2src/common/shared_ref_counter", diff --git a/mm2src/coins/Cargo.toml b/mm2src/coins/Cargo.toml index d541a1153b..617df41d6f 100644 --- a/mm2src/coins/Cargo.toml +++ b/mm2src/coins/Cargo.toml @@ -50,7 +50,6 @@ keys = { path = "../mm2_bitcoin/keys" } lazy_static = "1.4" libc = "0.2" lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } -lightning-background-processor = { path = "lightning_background_processor" } lightning-invoice = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } metrics = "0.12" mm2_core = { path = "../mm2_core" } @@ -99,7 +98,8 @@ web-sys = { version = "0.3.55", features = ["console", "Headers", "Request", "Re [target.'cfg(not(target_arch = "wasm32"))'.dependencies] dirs = { version = "1" } -lightning-persister = { path = "lightning_persister" } +lightning-background-processor = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } +lightning-persister = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } lightning-net-tokio = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } rust-ini = { version = "0.13" } rustls = { version = "0.20", features = ["dangerous_configuration"] } diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 08e2acbb00..d796d791d7 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -1,12 +1,18 @@ pub mod ln_conf; +mod ln_db; pub mod ln_errors; mod ln_events; +mod ln_filesystem_persister; mod ln_p2p; mod ln_platform; mod ln_serialization; +mod ln_sql; +mod ln_storage; mod ln_utils; use super::{lp_coinfind_or_err, DerivationMethod, MmCoinEnum}; +use crate::lightning::ln_events::init_events_abort_handlers; +use crate::lightning::ln_sql::SqliteLightningDB; use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, UtxoTxBuilder}; use crate::utxo::{sat_from_big_decimal, BlockchainNetwork, FeePolicy, GetUtxoListOps, UtxoTxGenerationOps}; @@ -23,7 +29,7 @@ use bitcrypto::dhash256; use bitcrypto::ChecksumType; use chain::TransactionOutput; use common::executor::spawn; -use common::log::{LogOnError, LogState}; +use common::log::{error, LogOnError, LogState}; use common::{async_blocking, calc_total_pages, log, now_ms, ten, PagingOptionsEnum}; use futures::{FutureExt, TryFutureExt}; use futures01::Future; @@ -39,11 +45,9 @@ use lightning_background_processor::BackgroundProcessor; use lightning_invoice::payment; use lightning_invoice::utils::{create_invoice_from_channelmanager, DefaultRouter}; use lightning_invoice::{Invoice, InvoiceDescription}; -use lightning_persister::storage::{ClosedChannelsFilter, DbStorage, FileSystemStorage, HTLCStatus, - NodesAddressesMapShared, PaymentInfo, PaymentType, PaymentsFilter, Scorer, - SqlChannelDetails}; -use lightning_persister::LightningPersister; use ln_conf::{ChannelOptions, LightningCoinConf, LightningProtocolConf, PlatformCoinConfirmations}; +use ln_db::{ClosedChannelsFilter, DBChannelDetails, DBPaymentInfo, DBPaymentsFilter, HTLCStatus, LightningDB, + PaymentType}; use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelError, CloseChannelResult, ConnectToNodeError, ConnectToNodeResult, EnableLightningError, EnableLightningResult, GenerateInvoiceError, GenerateInvoiceResult, GetChannelDetailsError, GetChannelDetailsResult, @@ -51,9 +55,11 @@ use ln_errors::{ClaimableBalancesError, ClaimableBalancesResult, CloseChannelErr ListPaymentsError, ListPaymentsResult, OpenChannelError, OpenChannelResult, SendPaymentError, SendPaymentResult}; use ln_events::LightningEventHandler; +use ln_filesystem_persister::{LightningFilesystemPersister, LightningPersisterShared}; use ln_p2p::{connect_to_node, ConnectToNodeRes, PeerManager}; use ln_platform::{h256_json_from_txid, Platform}; use ln_serialization::{InvoiceForRPC, NodeAddress, PublicKeyForRPC}; +use ln_storage::{LightningStorage, NodesAddressesMapShared, Scorer}; use ln_utils::{ChainMonitor, ChannelManager}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; @@ -81,7 +87,7 @@ pub struct LightningCoin { pub conf: LightningCoinConf, /// The lightning node peer manager that takes care of connecting to peers, etc.. pub peer_manager: Arc, - /// The lightning node background processor that takes care of tasks that need to happen periodically + /// The lightning node background processor that takes care of tasks that need to happen periodically. pub background_processor: Arc, /// The lightning node channel manager which keeps track of the number of open channels and sends messages to the appropriate /// channel, also tracks HTLC preimages and forwards onion packets appropriately. @@ -93,7 +99,9 @@ pub struct LightningCoin { /// The lightning node invoice payer. pub invoice_payer: Arc>>, /// The lightning node persister that takes care of writing/reading data from storage. - pub persister: Arc, + pub persister: LightningPersisterShared, + /// The lightning node db struct that takes care of reading/writing data from/to db. + pub db: SqliteLightningDB, /// The mutex storing the addresses of the nodes that the lightning node has open channels with, /// these addresses are used for reconnecting. pub open_channels_nodes: NodesAddressesMapShared, @@ -125,7 +133,7 @@ impl LightningCoin { }) } - fn pay_invoice(&self, invoice: Invoice) -> SendPaymentResult { + fn pay_invoice(&self, invoice: Invoice) -> SendPaymentResult { self.invoice_payer .pay_invoice(&invoice) .map_to_mm(|e| SendPaymentError::PaymentError(format!("{:?}", e)))?; @@ -138,17 +146,17 @@ impl LightningCoin { InvoiceDescription::Hash(h) => hex::encode(h.0.into_inner()), }; let payment_secret = Some(*invoice.payment_secret()); - Ok(PaymentInfo { + Ok(DBPaymentInfo { payment_hash, payment_type, description, preimage: None, secret: payment_secret, - amt_msat: invoice.amount_milli_satoshis(), + amt_msat: invoice.amount_milli_satoshis().map(|a| a as i64), fee_paid_msat: None, status: HTLCStatus::Pending, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, }) } @@ -157,7 +165,7 @@ impl LightningCoin { destination: PublicKey, amount_msat: u64, final_cltv_expiry_delta: u32, - ) -> SendPaymentResult { + ) -> SendPaymentResult { if final_cltv_expiry_delta < MIN_FINAL_CLTV_EXPIRY { return MmError::err(SendPaymentError::CLTVExpiryError( final_cltv_expiry_delta, @@ -171,17 +179,17 @@ impl LightningCoin { let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner()); let payment_type = PaymentType::OutboundPayment { destination }; - Ok(PaymentInfo { + Ok(DBPaymentInfo { payment_hash, payment_type, description: "".into(), preimage: Some(payment_preimage), secret: None, - amt_msat: Some(amount_msat), + amt_msat: Some(amount_msat as i64), fee_paid_msat: None, status: HTLCStatus::Pending, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, }) } @@ -621,28 +629,29 @@ pub async fn start_lightning( let logger = ctx.log.0.clone(); // Initialize Persister - let persister = ln_utils::init_persister(ctx, platform.clone(), conf.ticker.clone(), params.backup_path).await?; + let persister = ln_utils::init_persister(ctx, conf.ticker.clone(), params.backup_path).await?; // Initialize the KeysManager let keys_manager = ln_utils::init_keys_manager(ctx)?; // Initialize the NetGraphMsgHandler. This is used for providing routes to send payments over let network_graph = Arc::new(persister.get_network_graph(protocol_conf.network.into()).await?); - spawn(ln_utils::persist_network_graph_loop( - persister.clone(), - network_graph.clone(), - )); + let network_gossip = Arc::new(NetGraphMsgHandler::new( network_graph.clone(), None::>, logger.clone(), )); + // Initialize DB + let db = ln_utils::init_db(ctx, conf.ticker.clone()).await?; + // Initialize the ChannelManager let (chain_monitor, channel_manager) = ln_utils::init_channel_manager( platform.clone(), logger.clone(), persister.clone(), + db.clone(), keys_manager.clone(), conf.clone().into(), ) @@ -661,13 +670,15 @@ pub async fn start_lightning( ) .await?; + let events_abort_handlers = init_events_abort_handlers(platform.clone(), db.clone()).await?; + // Initialize the event handler let event_handler = Arc::new(ln_events::LightningEventHandler::new( - // It's safe to use unwrap here for now until implementing Native Client for Lightning platform.clone(), channel_manager.clone(), keys_manager.clone(), - persister.clone(), + db.clone(), + events_abort_handlers, )); // Initialize routing Scorer @@ -675,6 +686,12 @@ pub async fn start_lightning( spawn(ln_utils::persist_scorer_loop(persister.clone(), scorer.clone())); // Create InvoicePayer + // random_seed_bytes are additional random seed to improve privacy by adding a random CLTV expiry offset to each path's final hop. + // This helps obscure the intended recipient from adversarial intermediate hops. The seed is also used to randomize candidate paths during route selection. + // TODO: random_seed_bytes should be taken in consideration when implementing swaps because they change the payment lock-time. + // https://github.com/lightningdevkit/rust-lightning/issues/158 + // https://github.com/lightningdevkit/rust-lightning/pull/1286 + // https://github.com/lightningdevkit/rust-lightning/pull/1359 let router = DefaultRouter::new(network_graph, logger.clone(), keys_manager.get_secure_random_bytes()); let invoice_payer = Arc::new(InvoicePayer::new( channel_manager.clone(), @@ -685,17 +702,12 @@ pub async fn start_lightning( payment::RetryAttempts(params.payment_retries.unwrap_or(5)), )); - // Persist ChannelManager - // Note: if the ChannelManager is not persisted properly to disk, there is risk of channels force closing the next time LN starts up - let channel_manager_persister = persister.clone(); - let persist_channel_manager_callback = - move |node: &ChannelManager| channel_manager_persister.persist_manager(&*node); - // Start Background Processing. Runs tasks periodically in the background to keep LN node operational. // InvoicePayer will act as our event handler as it handles some of the payments related events before // delegating it to LightningEventHandler. + // note: background_processor stops automatically when dropped since BackgroundProcessor implements the Drop trait. let background_processor = Arc::new(BackgroundProcessor::start( - persist_channel_manager_callback, + persister.clone(), invoice_payer.clone(), chain_monitor.clone(), channel_manager.clone(), @@ -731,6 +743,7 @@ pub async fn start_lightning( keys_manager, invoice_payer, persister, + db, open_channels_nodes, }) } @@ -863,7 +876,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes user_config.own_channel_config.our_htlc_minimum_msat = min; } - let rpc_channel_id = ln_coin.persister.get_last_channel_rpc_id().await? as u64 + 1; + let rpc_channel_id = ln_coin.db.get_last_channel_rpc_id().await? as u64 + 1; let temp_channel_id = async_blocking(move || { channel_manager @@ -877,7 +890,7 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes unsigned_funding_txs.insert(rpc_channel_id, unsigned); } - let pending_channel_details = SqlChannelDetails::new( + let pending_channel_details = DBChannelDetails::new( rpc_channel_id, temp_channel_id, node_pubkey, @@ -892,7 +905,9 @@ pub async fn open_channel(ctx: MmArc, req: OpenChannelRequest) -> OpenChannelRes .save_nodes_addresses(ln_coin.open_channels_nodes) .await?; - ln_coin.persister.add_channel_to_db(pending_channel_details).await?; + if let Err(e) = ln_coin.db.add_channel_to_db(pending_channel_details).await { + error!("Unable to add new outbound channel {} to db: {}", rpc_channel_id, e); + } Ok(OpenChannelResponse { rpc_channel_id, @@ -1080,7 +1095,7 @@ pub struct ListClosedChannelsRequest { #[derive(Serialize)] pub struct ListClosedChannelsResponse { - closed_channels: Vec, + closed_channels: Vec, limit: usize, skipped: usize, total: usize, @@ -1098,7 +1113,7 @@ pub async fn list_closed_channels_by_filter( _ => return MmError::err(ListChannelsError::UnsupportedCoin(coin.ticker().to_string())), }; let closed_channels_res = ln_coin - .persister + .db .get_closed_channels_by_filter(req.filter, req.paging_options.clone(), req.limit) .await?; @@ -1122,7 +1137,7 @@ pub struct GetChannelDetailsRequest { #[serde(tag = "status", content = "details")] pub enum GetChannelDetailsResponse { Open(ChannelDetailsForRPC), - Closed(SqlChannelDetails), + Closed(DBChannelDetails), } pub async fn get_channel_details( @@ -1143,7 +1158,7 @@ pub async fn get_channel_details( Some(details) => GetChannelDetailsResponse::Open(details.into()), None => GetChannelDetailsResponse::Closed( ln_coin - .persister + .db .get_channel_from_db(req.rpc_channel_id) .await? .ok_or(GetChannelDetailsError::NoSuchChannel(req.rpc_channel_id))?, @@ -1194,19 +1209,19 @@ pub async fn generate_invoice( req.description.clone(), )?; let payment_hash = invoice.payment_hash().into_inner(); - let payment_info = PaymentInfo { + let payment_info = DBPaymentInfo { payment_hash: PaymentHash(payment_hash), payment_type: PaymentType::InboundPayment, description: req.description, preimage: None, secret: Some(*invoice.payment_secret()), - amt_msat: req.amount_in_msat, + amt_msat: req.amount_in_msat.map(|a| a as i64), fee_paid_msat: None, status: HTLCStatus::Pending, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, }; - ln_coin.persister.add_or_update_payment_in_db(payment_info).await?; + ln_coin.db.add_or_update_payment_in_db(payment_info).await?; Ok(GenerateInvoiceResponse { payment_hash: payment_hash.into(), invoice: invoice.into(), @@ -1265,10 +1280,7 @@ pub async fn send_payment(ctx: MmArc, req: SendPaymentReq) -> SendPaymentResult< expiry, } => ln_coin.keysend(destination.into(), amount_in_msat, expiry)?, }; - ln_coin - .persister - .add_or_update_payment_in_db(payment_info.clone()) - .await?; + ln_coin.db.add_or_update_payment_in_db(payment_info.clone()).await?; Ok(SendPaymentResponse { payment_hash: payment_info.payment_hash.0.into(), }) @@ -1287,18 +1299,27 @@ pub struct PaymentsFilterForRPC { pub to_timestamp: Option, } -impl From for PaymentsFilter { +impl From for DBPaymentsFilter { fn from(filter: PaymentsFilterForRPC) -> Self { - PaymentsFilter { - payment_type: filter.payment_type.map(From::from), + let (is_outbound, destination) = if let Some(payment_type) = filter.payment_type { + match payment_type { + PaymentTypeForRPC::OutboundPayment { destination } => (Some(true), Some(destination.0.to_string())), + PaymentTypeForRPC::InboundPayment => (Some(false), None), + } + } else { + (None, None) + }; + DBPaymentsFilter { + is_outbound, + destination, description: filter.description, - status: filter.status, - from_amount_msat: filter.from_amount_msat, - to_amount_msat: filter.to_amount_msat, - from_fee_paid_msat: filter.from_fee_paid_msat, - to_fee_paid_msat: filter.to_fee_paid_msat, - from_timestamp: filter.from_timestamp, - to_timestamp: filter.to_timestamp, + status: filter.status.map(|s| s.to_string()), + from_amount_msat: filter.from_amount_msat.map(|a| a as i64), + to_amount_msat: filter.to_amount_msat.map(|a| a as i64), + from_fee_paid_msat: filter.from_fee_paid_msat.map(|f| f as i64), + to_fee_paid_msat: filter.to_fee_paid_msat.map(|f| f as i64), + from_timestamp: filter.from_timestamp.map(|f| f as i64), + to_timestamp: filter.to_timestamp.map(|f| f as i64), } } } @@ -1350,16 +1371,16 @@ pub struct PaymentInfoForRPC { payment_type: PaymentTypeForRPC, description: String, #[serde(skip_serializing_if = "Option::is_none")] - amount_in_msat: Option, + amount_in_msat: Option, #[serde(skip_serializing_if = "Option::is_none")] - fee_paid_msat: Option, + fee_paid_msat: Option, status: HTLCStatus, - created_at: u64, - last_updated: u64, + created_at: i64, + last_updated: i64, } -impl From for PaymentInfoForRPC { - fn from(info: PaymentInfo) -> Self { +impl From for PaymentInfoForRPC { + fn from(info: DBPaymentInfo) -> Self { PaymentInfoForRPC { payment_hash: info.payment_hash.0.into(), payment_type: info.payment_type.into(), @@ -1390,7 +1411,7 @@ pub async fn list_payments_by_filter(ctx: MmArc, req: ListPaymentsReq) -> ListPa _ => return MmError::err(ListPaymentsError::UnsupportedCoin(coin.ticker().to_string())), }; let get_payments_res = ln_coin - .persister + .db .get_payments_by_filter( req.filter.map(From::from), req.paging_options.clone().map(|h| PaymentHash(h.0)), @@ -1429,11 +1450,7 @@ pub async fn get_payment_details( _ => return MmError::err(GetPaymentDetailsError::UnsupportedCoin(coin.ticker().to_string())), }; - if let Some(payment_info) = ln_coin - .persister - .get_payment_from_db(PaymentHash(req.payment_hash.0)) - .await? - { + if let Some(payment_info) = ln_coin.db.get_payment_from_db(PaymentHash(req.payment_hash.0)).await? { return Ok(GetPaymentDetailsResponse { payment_details: payment_info.into(), }); diff --git a/mm2src/coins/lightning/ln_conf.rs b/mm2src/coins/lightning/ln_conf.rs index f69a9bca10..b40234d32d 100644 --- a/mm2src/coins/lightning/ln_conf.rs +++ b/mm2src/coins/lightning/ln_conf.rs @@ -124,6 +124,11 @@ pub struct OurChannelsConfig { /// The smallest value HTLC we will accept to process. The channel gets closed any time /// our counterparty misbehaves by sending us an HTLC with a value smaller than this. pub our_htlc_minimum_msat: Option, + /// If set, we attempt to negotiate the `scid_privacy` (referred to as `scid_alias` in the + /// BOLTs) option for outbound private channels. This provides better privacy by not including + /// our real on-chain channel UTXO in each invoice and requiring that our counterparty only + /// relay HTLCs to us using the channel's SCID alias. + pub negotiate_scid_privacy: Option, } impl From for ChannelHandshakeConfig { @@ -142,6 +147,10 @@ impl From for ChannelHandshakeConfig { channel_handshake_config.our_htlc_minimum_msat = min; } + if let Some(scid_privacy) = config.negotiate_scid_privacy { + channel_handshake_config.negotiate_scid_privacy = scid_privacy + } + channel_handshake_config } } diff --git a/mm2src/coins/lightning_persister/src/storage.rs b/mm2src/coins/lightning/ln_db.rs similarity index 68% rename from mm2src/coins/lightning_persister/src/storage.rs rename to mm2src/coins/lightning/ln_db.rs index c0b9ac7e9d..b47a72361a 100644 --- a/mm2src/coins/lightning_persister/src/storage.rs +++ b/mm2src/coins/lightning/ln_db.rs @@ -1,53 +1,21 @@ use async_trait::async_trait; -use bitcoin::Network; use common::{now_ms, PagingOptionsEnum}; use db_common::sqlite::rusqlite::types::FromSqlError; use derive_more::Display; use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; -use lightning::routing::network_graph::NetworkGraph; -use lightning::routing::scoring::ProbabilisticScorer; -use parking_lot::Mutex as PaMutex; use secp256k1::PublicKey; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::net::SocketAddr; use std::str::FromStr; -use std::sync::{Arc, Mutex}; - -pub type NodesAddressesMap = HashMap; -pub type NodesAddressesMapShared = Arc>; -pub type Scorer = ProbabilisticScorer>; -#[async_trait] -pub trait FileSystemStorage { - type Error; - - /// Initializes dirs/collection/tables in storage for a specified coin - async fn init_fs(&self) -> Result<(), Self::Error>; - - async fn is_fs_initialized(&self) -> Result; - - async fn get_nodes_addresses(&self) -> Result, Self::Error>; - - async fn save_nodes_addresses(&self, nodes_addresses: NodesAddressesMapShared) -> Result<(), Self::Error>; - - async fn get_network_graph(&self, network: Network) -> Result; - - async fn save_network_graph(&self, network_graph: Arc) -> Result<(), Self::Error>; - - async fn get_scorer(&self, network_graph: Arc) -> Result; - - async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error>; -} #[derive(Clone, Debug, PartialEq, Serialize)] -pub struct SqlChannelDetails { - pub rpc_id: u64, +pub struct DBChannelDetails { + pub rpc_id: i64, pub channel_id: String, pub counterparty_node_id: String, #[serde(skip_serializing_if = "Option::is_none")] pub funding_tx: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub funding_value: Option, + pub funding_value: Option, #[serde(skip_serializing_if = "Option::is_none")] pub closing_tx: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -57,16 +25,16 @@ pub struct SqlChannelDetails { #[serde(skip_serializing_if = "Option::is_none")] pub claimed_balance: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub funding_generated_in_block: Option, + pub funding_generated_in_block: Option, pub is_outbound: bool, pub is_public: bool, pub is_closed: bool, - pub created_at: u64, + pub created_at: i64, #[serde(skip_serializing_if = "Option::is_none")] - pub closed_at: Option, + pub closed_at: Option, } -impl SqlChannelDetails { +impl DBChannelDetails { #[inline] pub fn new( rpc_id: u64, @@ -75,8 +43,8 @@ impl SqlChannelDetails { is_outbound: bool, is_public: bool, ) -> Self { - SqlChannelDetails { - rpc_id, + DBChannelDetails { + rpc_id: rpc_id as i64, channel_id: hex::encode(channel_id), counterparty_node_id: counterparty_node_id.to_string(), funding_tx: None, @@ -89,7 +57,7 @@ impl SqlChannelDetails { is_outbound, is_public, is_closed: false, - created_at: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, closed_at: None, } } @@ -112,8 +80,8 @@ pub struct ClosedChannelsFilter { pub channel_id: Option, pub counterparty_node_id: Option, pub funding_tx: Option, - pub from_funding_value: Option, - pub to_funding_value: Option, + pub from_funding_value: Option, + pub to_funding_value: Option, pub closing_tx: Option, pub closure_reason: Option, pub claiming_tx: Option, @@ -124,7 +92,7 @@ pub struct ClosedChannelsFilter { } pub struct GetClosedChannelsResult { - pub channels: Vec, + pub channels: Vec, pub skipped: usize, pub total: usize, } @@ -157,40 +125,41 @@ pub enum PaymentType { } #[derive(Clone, Debug, PartialEq)] -pub struct PaymentInfo { +pub struct DBPaymentInfo { pub payment_hash: PaymentHash, pub payment_type: PaymentType, pub description: String, pub preimage: Option, pub secret: Option, - pub amt_msat: Option, - pub fee_paid_msat: Option, + pub amt_msat: Option, + pub fee_paid_msat: Option, pub status: HTLCStatus, - pub created_at: u64, - pub last_updated: u64, + pub created_at: i64, + pub last_updated: i64, } #[derive(Clone)] -pub struct PaymentsFilter { - pub payment_type: Option, +pub struct DBPaymentsFilter { + pub is_outbound: Option, + pub destination: Option, pub description: Option, - pub status: Option, - pub from_amount_msat: Option, - pub to_amount_msat: Option, - pub from_fee_paid_msat: Option, - pub to_fee_paid_msat: Option, - pub from_timestamp: Option, - pub to_timestamp: Option, + pub status: Option, + pub from_amount_msat: Option, + pub to_amount_msat: Option, + pub from_fee_paid_msat: Option, + pub to_fee_paid_msat: Option, + pub from_timestamp: Option, + pub to_timestamp: Option, } pub struct GetPaymentsResult { - pub payments: Vec, + pub payments: Vec, pub skipped: usize, pub total: usize, } #[async_trait] -pub trait DbStorage { +pub trait LightningDB { type Error; /// Initializes tables in DB. @@ -204,35 +173,37 @@ pub trait DbStorage { /// Inserts a new channel record in the DB. The record's data is completed using add_funding_tx_to_db, /// add_closing_tx_to_db, add_claiming_tx_to_db when this information is available. - async fn add_channel_to_db(&self, details: SqlChannelDetails) -> Result<(), Self::Error>; + async fn add_channel_to_db(&self, details: DBChannelDetails) -> Result<(), Self::Error>; /// Updates a channel's DB record with the channel's funding transaction information. async fn add_funding_tx_to_db( &self, - rpc_id: u64, + rpc_id: i64, funding_tx: String, - funding_value: u64, - funding_generated_in_block: u64, + funding_value: i64, + funding_generated_in_block: i64, ) -> Result<(), Self::Error>; /// Updates funding_tx_block_height value for a channel in the DB. Should be used to update the block height of /// the funding tx when the transaction is confirmed on-chain. - async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: u64) -> Result<(), Self::Error>; + async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: i64) -> Result<(), Self::Error>; /// Updates the is_closed value for a channel in the DB to 1. async fn update_channel_to_closed( &self, - rpc_id: u64, + rpc_id: i64, closure_reason: String, - close_at: u64, + close_at: i64, ) -> Result<(), Self::Error>; - /// Gets the list of closed channels records in the DB with no closing tx hashs saved yet. Can be used to check if - /// the closing tx hash needs to be fetched from the chain and saved to DB when initializing the persister. - async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error>; + /// Gets the list of closed channels records in the DB that have funding tx hashes saved with no closing + /// tx hashes saved yet. + /// Can be used to check if the closing tx hash needs to be fetched from the chain and saved to DB + /// when initializing the persister. + async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error>; /// Updates a channel's DB record with the channel's closing transaction hash. - async fn add_closing_tx_to_db(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error>; + async fn add_closing_tx_to_db(&self, rpc_id: i64, closing_tx: String) -> Result<(), Self::Error>; /// Updates a channel's DB record with information about the transaction responsible for claiming the channel's /// closing balance back to the user's address. @@ -244,7 +215,7 @@ pub trait DbStorage { ) -> Result<(), Self::Error>; /// Gets a channel record from DB by the channel's rpc_id. - async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error>; + async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error>; /// Gets the list of closed channels that match the provided filter criteria. The number of requested records is /// specified by the limit parameter, the starting record to list from is specified by the paging parameter. The @@ -257,17 +228,17 @@ pub trait DbStorage { ) -> Result; /// Inserts or updates a new payment record in the DB. - async fn add_or_update_payment_in_db(&self, info: PaymentInfo) -> Result<(), Self::Error>; + async fn add_or_update_payment_in_db(&self, info: DBPaymentInfo) -> Result<(), Self::Error>; /// Gets a payment's record from DB by the payment's hash. - async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error>; + async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error>; /// Gets the list of payments that match the provided filter criteria. The number of requested records is specified /// by the limit parameter, the starting record to list from is specified by the paging parameter. The total number /// of matched records along with the number of skipped records are also returned in the result. async fn get_payments_by_filter( &self, - filter: Option, + filter: Option, paging: PagingOptionsEnum, limit: usize, ) -> Result; diff --git a/mm2src/coins/lightning/ln_errors.rs b/mm2src/coins/lightning/ln_errors.rs index 72b581f647..e70f555d8c 100644 --- a/mm2src/coins/lightning/ln_errors.rs +++ b/mm2src/coins/lightning/ln_errors.rs @@ -1,8 +1,6 @@ use crate::utxo::rpc_clients::UtxoRpcError; use crate::utxo::GenerateTxError; use crate::{BalanceError, CoinFindError, NumConversError, PrivKeyNotAllowed, UnexpectedDerivationMethod}; -use bitcoin::consensus::encode; -use common::jsonrpc_client::JsonRpcError; use common::HttpStatusCode; use db_common::sqlite::rusqlite::Error as SqlError; use derive_more::Display; @@ -10,6 +8,7 @@ use http::StatusCode; use lightning_invoice::SignOrCreationError; use mm2_err_handle::prelude::*; use rpc::v1::types::H256 as H256Json; +use std::num::TryFromIntError; use utxo_signer::with_key_pair::UtxoSignWithKeyPairError; pub type EnableLightningResult = Result>; @@ -42,8 +41,6 @@ pub enum EnableLightningError { InvalidPath(String), #[display(fmt = "System time error {}", _0)] SystemTimeError(String), - #[display(fmt = "Hash error {}", _0)] - HashError(String), #[display(fmt = "RPC error {}", _0)] RpcError(String), #[display(fmt = "DB error {}", _0)] @@ -60,7 +57,6 @@ impl HttpStatusCode for EnableLightningError { | EnableLightningError::InvalidPath(_) | EnableLightningError::SystemTimeError(_) | EnableLightningError::IOError(_) - | EnableLightningError::HashError(_) | EnableLightningError::ConnectToNodeError(_) | EnableLightningError::InvalidConfiguration(_) | EnableLightningError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, @@ -493,7 +489,7 @@ impl From for ClaimableBalancesError { } } -#[derive(Display)] +#[derive(Display, PartialEq)] pub enum SaveChannelClosingError { #[display(fmt = "DB error: {}", _0)] DbError(String), @@ -507,55 +503,14 @@ pub enum SaveChannelClosingError { FundingTxParseError(String), #[display(fmt = "Error while waiting for the funding transaction to be spent: {}", _0)] WaitForFundingTxSpendError(String), + #[display(fmt = "Error while converting types: {}", _0)] + ConversionError(TryFromIntError), } impl From for SaveChannelClosingError { fn from(err: SqlError) -> SaveChannelClosingError { SaveChannelClosingError::DbError(err.to_string()) } } -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum GetTxError { - Rpc(UtxoRpcError), - TxDeserialization(encode::Error), -} - -impl From for GetTxError { - fn from(err: UtxoRpcError) -> GetTxError { GetTxError::Rpc(err) } -} - -impl From for GetTxError { - fn from(err: encode::Error) -> GetTxError { GetTxError::TxDeserialization(err) } -} - -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum GetHeaderError { - Rpc(JsonRpcError), - HeaderDeserialization(encode::Error), -} - -impl From for GetHeaderError { - fn from(err: JsonRpcError) -> GetHeaderError { GetHeaderError::Rpc(err) } -} - -impl From for GetHeaderError { - fn from(err: encode::Error) -> GetHeaderError { GetHeaderError::HeaderDeserialization(err) } -} - -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum FindWatchedOutputSpendError { - HashNotHeight, - DeserializationErr(encode::Error), - RpcError(String), - GetHeaderError(GetHeaderError), -} - -impl From for FindWatchedOutputSpendError { - fn from(err: JsonRpcError) -> Self { FindWatchedOutputSpendError::RpcError(err.to_string()) } -} - -impl From for FindWatchedOutputSpendError { - fn from(err: encode::Error) -> Self { FindWatchedOutputSpendError::DeserializationErr(err) } +impl From for SaveChannelClosingError { + fn from(err: TryFromIntError) -> SaveChannelClosingError { SaveChannelClosingError::ConversionError(err) } } diff --git a/mm2src/coins/lightning/ln_events.rs b/mm2src/coins/lightning/ln_events.rs index 3f898e4fa3..59af6a94ee 100644 --- a/mm2src/coins/lightning/ln_events.rs +++ b/mm2src/coins/lightning/ln_events.rs @@ -1,14 +1,18 @@ use super::*; +use crate::lightning::ln_db::{DBChannelDetails, HTLCStatus, LightningDB, PaymentType}; use crate::lightning::ln_errors::{SaveChannelClosingError, SaveChannelClosingResult}; +use crate::lightning::ln_sql::SqliteLightningDB; use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::Transaction; +use bitcoin::consensus::encode::serialize_hex; use common::executor::{spawn, Timer}; use common::log::{error, info}; -use common::now_ms; +use common::{now_ms, spawn_abortable, AbortOnDropHandle}; use core::time::Duration; -use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}; +use lightning::chain::chaininterface::{ConfirmationTarget, FeeEstimator}; use lightning::chain::keysinterface::SpendableOutputDescriptor; use lightning::util::events::{Event, EventHandler, PaymentPurpose}; +use parking_lot::Mutex as PaMutex; use rand::Rng; use script::{Builder, SignatureVersion}; use secp256k1::Secp256k1; @@ -22,7 +26,8 @@ pub struct LightningEventHandler { platform: Arc, channel_manager: Arc, keys_manager: Arc, - persister: Arc, + db: SqliteLightningDB, + abort_handlers: Arc>>, } impl EventHandler for LightningEventHandler { @@ -114,22 +119,41 @@ impl EventHandler for LightningEventHandler { funding_satoshis, push_msat, channel_type: _, - } => { - info!( - "Handling OpenChannelRequest from node: {} with funding value: {} and starting balance: {}", - counterparty_node_id, - funding_satoshis, - push_msat, - ); - if self.channel_manager.accept_inbound_channel(temporary_channel_id, 0).is_ok() { - // Todo: once the rust-lightning PR for user_channel_id in accept_inbound_channel is released - // use user_channel_id to get the funding tx here once the funding tx is available. - } - }, + } => self.handle_open_channel_request(*temporary_channel_id, *counterparty_node_id, *funding_satoshis, *push_msat), } } } +pub async fn init_events_abort_handlers( + platform: Arc, + db: SqliteLightningDB, +) -> EnableLightningResult>>> { + let abort_handlers = Arc::new(PaMutex::new(Vec::new())); + let closed_channels_without_closing_tx = db.get_closed_channels_with_no_closing_tx().await?; + for channel_details in closed_channels_without_closing_tx { + let platform = platform.clone(); + let db = db.clone(); + let user_channel_id = channel_details.rpc_id; + let abort_handler = spawn_abortable(async move { + if let Ok(closing_tx_hash) = platform + .get_channel_closing_tx(channel_details) + .await + .error_log_passthrough() + { + if let Err(e) = db.add_closing_tx_to_db(user_channel_id, closing_tx_hash).await { + log::error!( + "Unable to update channel {} closing details in DB: {}", + user_channel_id, + e + ); + } + } + }); + abort_handlers.lock().push(abort_handler); + } + Ok(abort_handlers) +} + // Generates the raw funding transaction with one output equal to the channel value. fn sign_funding_transaction( user_channel_id: u64, @@ -167,39 +191,55 @@ fn sign_funding_transaction( } async fn save_channel_closing_details( - persister: Arc, + db: SqliteLightningDB, platform: Arc, user_channel_id: u64, reason: String, ) -> SaveChannelClosingResult<()> { - persister - .update_channel_to_closed(user_channel_id, reason, now_ms() / 1000) + db.update_channel_to_closed(user_channel_id as i64, reason, (now_ms() / 1000) as i64) .await?; - let channel_details = persister + let channel_details = db .get_channel_from_db(user_channel_id) .await? .ok_or_else(|| MmError::new(SaveChannelClosingError::ChannelNotFound(user_channel_id)))?; let closing_tx_hash = platform.get_channel_closing_tx(channel_details).await?; - persister.add_closing_tx_to_db(user_channel_id, closing_tx_hash).await?; + db.add_closing_tx_to_db(user_channel_id as i64, closing_tx_hash).await?; Ok(()) } +async fn add_claiming_tx_to_db_loop( + db: SqliteLightningDB, + closing_txid: String, + claiming_txid: String, + claimed_balance: f64, +) { + while let Err(e) = db + .add_claiming_tx_to_db(closing_txid.clone(), claiming_txid.clone(), claimed_balance) + .await + { + error!("error {}", e); + Timer::sleep(TRY_LOOP_INTERVAL).await; + } +} + impl LightningEventHandler { pub fn new( platform: Arc, channel_manager: Arc, keys_manager: Arc, - persister: Arc, + db: SqliteLightningDB, + abort_handlers: Arc>>, ) -> Self { LightningEventHandler { platform, channel_manager, keys_manager, - persister, + db, + abort_handlers, } } @@ -235,18 +275,17 @@ impl LightningEventHandler { return; } let platform = self.platform.clone(); - let persister = self.persister.clone(); + let db = self.db.clone(); spawn(async move { let best_block_height = platform.best_block_height(); - persister - .add_funding_tx_to_db( - user_channel_id, - funding_txid.to_string(), - channel_value_satoshis, - best_block_height, - ) - .await - .error_log(); + db.add_funding_tx_to_db( + user_channel_id as i64, + funding_txid.to_string(), + channel_value_satoshis as i64, + best_block_height as i64, + ) + .await + .error_log(); }); } @@ -276,38 +315,34 @@ impl LightningEventHandler { }, false => HTLCStatus::Failed, }; - let persister = self.persister.clone(); + let db = self.db.clone(); match purpose { PaymentPurpose::InvoicePayment { .. } => spawn(async move { - if let Ok(Some(mut payment_info)) = persister - .get_payment_from_db(payment_hash) - .await - .error_log_passthrough() - { + if let Ok(Some(mut payment_info)) = db.get_payment_from_db(payment_hash).await.error_log_passthrough() { payment_info.preimage = Some(payment_preimage); payment_info.status = HTLCStatus::Succeeded; - payment_info.amt_msat = Some(amt); - payment_info.last_updated = now_ms() / 1000; - if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { + payment_info.amt_msat = Some(amt as i64); + payment_info.last_updated = (now_ms() / 1000) as i64; + if let Err(e) = db.add_or_update_payment_in_db(payment_info).await { error!("Unable to update payment information in DB: {}", e); } } }), PaymentPurpose::SpontaneousPayment(_) => { - let payment_info = PaymentInfo { + let payment_info = DBPaymentInfo { payment_hash, payment_type: PaymentType::InboundPayment, description: "".into(), preimage: Some(payment_preimage), secret: payment_secret, - amt_msat: Some(amt), + amt_msat: Some(amt as i64), fee_paid_msat: None, status, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, }; spawn(async move { - if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { + if let Err(e) = db.add_or_update_payment_in_db(payment_info).await { error!("Unable to update payment information in DB: {}", e); } }); @@ -325,19 +360,15 @@ impl LightningEventHandler { "Handling PaymentSent event for payment_hash: {}", hex::encode(payment_hash.0) ); - let persister = self.persister.clone(); + let db = self.db.clone(); spawn(async move { - if let Ok(Some(mut payment_info)) = persister - .get_payment_from_db(payment_hash) - .await - .error_log_passthrough() - { + if let Ok(Some(mut payment_info)) = db.get_payment_from_db(payment_hash).await.error_log_passthrough() { payment_info.preimage = Some(payment_preimage); payment_info.status = HTLCStatus::Succeeded; - payment_info.fee_paid_msat = fee_paid_msat; - payment_info.last_updated = now_ms() / 1000; + payment_info.fee_paid_msat = fee_paid_msat.map(|f| f as i64); + payment_info.last_updated = (now_ms() / 1000) as i64; let amt_msat = payment_info.amt_msat; - if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { + if let Err(e) = db.add_or_update_payment_in_db(payment_info).await { error!("Unable to update payment information in DB: {}", e); } info!( @@ -355,21 +386,20 @@ impl LightningEventHandler { hex::encode(channel_id), reason ); - let persister = self.persister.clone(); + let db = self.db.clone(); let platform = self.platform.clone(); - // Todo: Handle inbound channels closure case after updating to latest version of rust-lightning - // as it has a new OpenChannelRequest event where we can give an inbound channel a user_channel_id - // other than 0 in sql - if user_channel_id != 0 { - spawn(async move { - if let Err(e) = save_channel_closing_details(persister, platform, user_channel_id, reason).await { + let abort_handler = spawn_abortable(async move { + if let Err(e) = save_channel_closing_details(db, platform, user_channel_id, reason).await { + // This is the case when a channel is closed before funding is broadcasted due to the counterparty disconnecting or other incompatibility issue. + if e != SaveChannelClosingError::FundingTxNull.into() { error!( "Unable to update channel {} closing details in DB: {}", user_channel_id, e ); } - }); - } + } + }); + self.abort_handlers.lock().push(abort_handler); } fn handle_payment_failed(&self, payment_hash: PaymentHash) { @@ -377,16 +407,12 @@ impl LightningEventHandler { "Handling PaymentFailed event for payment_hash: {}", hex::encode(payment_hash.0) ); - let persister = self.persister.clone(); + let db = self.db.clone(); spawn(async move { - if let Ok(Some(mut payment_info)) = persister - .get_payment_from_db(payment_hash) - .await - .error_log_passthrough() - { + if let Ok(Some(mut payment_info)) = db.get_payment_from_db(payment_hash).await.error_log_passthrough() { payment_info.status = HTLCStatus::Failed; - payment_info.last_updated = now_ms() / 1000; - if let Err(e) = persister.add_or_update_payment_in_db(payment_info).await { + payment_info.last_updated = (now_ms() / 1000) as i64; + if let Err(e) = db.add_or_update_payment_in_db(payment_info).await { error!("Unable to update payment information in DB: {}", e); } } @@ -418,7 +444,7 @@ impl LightningEventHandler { let change_destination_script = Builder::build_witness_script(&my_address.hash).to_bytes().take().into(); let feerate_sat_per_1000_weight = self.platform.get_est_sat_per_1000_weight(ConfirmationTarget::Normal); let output_descriptors = &outputs.iter().collect::>(); - let spending_tx = match self.keys_manager.spend_spendable_outputs( + let claiming_tx = match self.keys_manager.spend_spendable_outputs( output_descriptors, Vec::new(), change_destination_script, @@ -432,12 +458,23 @@ impl LightningEventHandler { }, }; + let claiming_txid = claiming_tx.txid(); + let tx_hex = serialize_hex(&claiming_tx); + if let Err(e) = tokio::task::block_in_place(move || self.platform.coin.send_raw_tx(&tx_hex).wait()) { + // TODO: broadcast transaction through p2p network in this case + error!( + "Broadcasting of the claiming transaction {} failed: {}", + claiming_txid, e + ); + return; + } + let claiming_tx_inputs_value = outputs.iter().fold(0, |sum, output| match output { SpendableOutputDescriptor::StaticOutput { output, .. } => sum + output.value, SpendableOutputDescriptor::DelayedPaymentOutput(descriptor) => sum + descriptor.output.value, SpendableOutputDescriptor::StaticPaymentOutput(descriptor) => sum + descriptor.output.value, }); - let claiming_tx_outputs_value = spending_tx.output.iter().fold(0, |sum, txout| sum + txout.value); + let claiming_tx_outputs_value = claiming_tx.output.iter().fold(0, |sum, txout| sum + txout.value); if claiming_tx_inputs_value < claiming_tx_outputs_value { error!( "Claiming transaction input value {} can't be less than outputs value {}!", @@ -460,22 +497,90 @@ impl LightningEventHandler { (descriptor.outpoint.txid.to_string(), descriptor.output.value) }, }; - let claiming_txid = spending_tx.txid().to_string(); - let persister = self.persister.clone(); - spawn(async move { - ok_or_retry_after_sleep!( - persister - .add_claiming_tx_to_db( - closing_txid.clone(), - claiming_txid.clone(), - (claimed_balance as f64) - claiming_tx_fee_per_channel, - ) - .await, - TRY_LOOP_INTERVAL - ); - }); - - self.platform.broadcast_transaction(&spending_tx); + let db = self.db.clone(); + + // This doesn't need to be respawned on restart unlike add_closing_tx_to_db since Event::SpendableOutputs will be re-fired on restart + // if the spending_tx is not broadcasted. + let abort_handler = spawn_abortable(add_claiming_tx_to_db_loop( + db, + closing_txid, + claiming_txid.to_string(), + (claimed_balance as f64) - claiming_tx_fee_per_channel, + )); + self.abort_handlers.lock().push(abort_handler); } } + + fn handle_open_channel_request( + &self, + temporary_channel_id: [u8; 32], + counterparty_node_id: PublicKey, + funding_satoshis: u64, + push_msat: u64, + ) { + info!( + "Handling OpenChannelRequest from node: {} with funding value: {} and starting balance: {}", + counterparty_node_id, funding_satoshis, push_msat, + ); + + let db = self.db.clone(); + let channel_manager = self.channel_manager.clone(); + let platform = self.platform.clone(); + spawn(async move { + if let Ok(last_channel_rpc_id) = db.get_last_channel_rpc_id().await.error_log_passthrough() { + let user_channel_id = last_channel_rpc_id as u64 + 1; + if channel_manager + .accept_inbound_channel(&temporary_channel_id, user_channel_id) + .is_ok() + { + let is_public = match channel_manager + .list_channels() + .into_iter() + .find(|chan| chan.user_channel_id == user_channel_id) + { + Some(details) => details.is_public, + None => { + error!( + "Inbound channel {} details should be found by list_channels!", + user_channel_id + ); + return; + }, + }; + + let pending_channel_details = DBChannelDetails::new( + user_channel_id, + temporary_channel_id, + counterparty_node_id, + false, + is_public, + ); + if let Err(e) = db.add_channel_to_db(pending_channel_details).await { + error!("Unable to add new inbound channel {} to db: {}", user_channel_id, e); + } + + while let Some(details) = channel_manager + .list_channels() + .into_iter() + .find(|chan| chan.user_channel_id == user_channel_id) + { + if let Some(funding_tx) = details.funding_txo { + let best_block_height = platform.best_block_height(); + db.add_funding_tx_to_db( + user_channel_id as i64, + funding_tx.txid.to_string(), + funding_satoshis as i64, + best_block_height as i64, + ) + .await + .error_log(); + break; + } + + Timer::sleep(TRY_LOOP_INTERVAL).await; + } + } + } + }); + } } diff --git a/mm2src/coins/lightning/ln_filesystem_persister.rs b/mm2src/coins/lightning/ln_filesystem_persister.rs new file mode 100644 index 0000000000..7182f51eb3 --- /dev/null +++ b/mm2src/coins/lightning/ln_filesystem_persister.rs @@ -0,0 +1,399 @@ +use crate::lightning::ln_platform::Platform; +use crate::lightning::ln_storage::{LightningStorage, NodesAddressesMap, NodesAddressesMapShared, Scorer}; +use crate::lightning::ln_utils::{ChainMonitor, ChannelManager}; +use async_trait::async_trait; +use bitcoin::blockdata::constants::genesis_block; +use bitcoin::Network; +use bitcoin_hashes::hex::ToHex; +use common::async_blocking; +use common::log::LogState; +use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; +use lightning::chain::keysinterface::{InMemorySigner, KeysManager, Sign}; +use lightning::chain::transaction::OutPoint; +use lightning::chain::{chainmonitor, ChannelMonitorUpdateErr}; +use lightning::routing::network_graph::NetworkGraph; +use lightning::routing::scoring::ProbabilisticScoringParameters; +use lightning::util::ser::{Readable, ReadableArgs, Writeable}; +use lightning_background_processor::Persister; +use lightning_persister::FilesystemPersister; +use mm2_io::fs::check_dir_operations; +use secp256k1::PublicKey; +use std::collections::HashMap; +use std::fs; +use std::io::{BufReader, BufWriter}; +use std::net::SocketAddr; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; + +#[cfg(target_family = "unix")] use std::os::unix::io::AsRawFd; + +#[cfg(target_family = "windows")] +use {std::ffi::OsStr, std::os::windows::ffi::OsStrExt}; + +pub struct LightningFilesystemPersister { + main_path: PathBuf, + backup_path: Option, + channels_persister: FilesystemPersister, +} + +impl LightningFilesystemPersister { + /// Initialize a new LightningPersister and set the path to the individual channels' + /// files. + #[inline] + pub fn new(main_path: PathBuf, backup_path: Option) -> Self { + Self { + main_path: main_path.clone(), + backup_path, + channels_persister: FilesystemPersister::new(main_path.display().to_string()), + } + } + + /// Get the directory which was provided when this persister was initialized. + #[inline] + pub fn main_path(&self) -> PathBuf { self.main_path.clone() } + + /// Get the backup directory which was provided when this persister was initialized. + #[inline] + pub fn backup_path(&self) -> Option { self.backup_path.clone() } + + /// Get the channels_persister which was initialized when this persister was initialized. + #[inline] + pub fn channels_persister(&self) -> &FilesystemPersister { &self.channels_persister } + + pub fn monitor_backup_path(&self) -> Option { + if let Some(mut backup_path) = self.backup_path() { + backup_path.push("monitors"); + return Some(backup_path); + } + None + } + + pub fn nodes_addresses_path(&self) -> PathBuf { + let mut path = self.main_path(); + path.push("channel_nodes_data"); + path + } + + pub fn nodes_addresses_backup_path(&self) -> Option { + if let Some(mut backup_path) = self.backup_path() { + backup_path.push("channel_nodes_data"); + return Some(backup_path); + } + None + } + + pub fn network_graph_path(&self) -> PathBuf { + let mut path = self.main_path(); + path.push("network_graph"); + path + } + + pub fn scorer_path(&self) -> PathBuf { + let mut path = self.main_path(); + path.push("scorer"); + path + } + + pub fn manager_path(&self) -> PathBuf { + let mut path = self.main_path(); + path.push("manager"); + path + } +} + +#[derive(Clone)] +pub struct LightningPersisterShared(pub Arc); + +impl Deref for LightningPersisterShared { + type Target = LightningFilesystemPersister; + fn deref(&self) -> &LightningFilesystemPersister { self.0.deref() } +} + +impl Persister, Arc, Arc, Arc, Arc> + for LightningPersisterShared +{ + fn persist_manager(&self, channel_manager: &ChannelManager) -> Result<(), std::io::Error> { + FilesystemPersister::persist_manager(self.0.main_path().display().to_string(), channel_manager)?; + if let Some(backup_path) = self.0.backup_path() { + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(backup_path)?; + channel_manager.write(&mut BufWriter::new(file))?; + } + Ok(()) + } + + fn persist_graph(&self, network_graph: &NetworkGraph) -> Result<(), std::io::Error> { + if FilesystemPersister::persist_network_graph(self.0.main_path().display().to_string(), network_graph).is_err() + { + // Persistence errors here are non-fatal as we can just fetch the routing graph + // again later, but they may indicate a disk error which could be fatal elsewhere. + eprintln!("Warning: Failed to persist network graph, check your disk and permissions"); + } + + Ok(()) + } +} + +#[cfg(target_family = "windows")] +macro_rules! call { + ($e: expr) => { + if $e != 0 { + return Ok(()); + } else { + return Err(std::io::Error::last_os_error()); + } + }; +} + +#[cfg(target_family = "windows")] +fn path_to_windows_str>(path: T) -> Vec { + path.as_ref().encode_wide().chain(Some(0)).collect() +} + +fn write_monitor_to_file( + mut path: PathBuf, + filename: String, + monitor: &ChannelMonitor, +) -> std::io::Result<()> { + // Do a crazy dance with lots of fsync()s to be overly cautious here... + // We never want to end up in a state where we've lost the old data, or end up using the + // old data on power loss after we've returned. + // The way to atomically write a file on Unix platforms is: + // open(tmpname), write(tmpfile), fsync(tmpfile), close(tmpfile), rename(), fsync(dir) + path.push(filename); + let filename_with_path = path.display().to_string(); + let tmp_filename = format!("{}.tmp", filename_with_path); + + { + let mut f = fs::File::create(&tmp_filename)?; + monitor.write(&mut f)?; + f.sync_all()?; + } + // Fsync the parent directory on Unix. + #[cfg(target_family = "unix")] + { + fs::rename(&tmp_filename, &filename_with_path)?; + let path = Path::new(&filename_with_path).parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("can't find parent dir for {}", filename_with_path), + ) + })?; + let dir_file = fs::OpenOptions::new().read(true).open(path)?; + unsafe { + libc::fsync(dir_file.as_raw_fd()); + } + } + #[cfg(target_family = "windows")] + { + let src = PathBuf::from(tmp_filename); + let dst = PathBuf::from(filename_with_path.clone()); + if Path::new(&filename_with_path).exists() { + unsafe { + winapi::um::winbase::ReplaceFileW( + path_to_windows_str(dst).as_ptr(), + path_to_windows_str(src).as_ptr(), + std::ptr::null(), + winapi::um::winbase::REPLACEFILE_IGNORE_MERGE_ERRORS, + std::ptr::null_mut() as *mut winapi::ctypes::c_void, + std::ptr::null_mut() as *mut winapi::ctypes::c_void, + ) + }; + } else { + call!(unsafe { + winapi::um::winbase::MoveFileExW( + path_to_windows_str(src).as_ptr(), + path_to_windows_str(dst).as_ptr(), + winapi::um::winbase::MOVEFILE_WRITE_THROUGH | winapi::um::winbase::MOVEFILE_REPLACE_EXISTING, + ) + }); + } + } + Ok(()) +} + +impl chainmonitor::Persist for LightningFilesystemPersister { + fn persist_new_channel( + &self, + funding_txo: OutPoint, + monitor: &ChannelMonitor, + update_id: chainmonitor::MonitorUpdateId, + ) -> Result<(), ChannelMonitorUpdateErr> { + self.channels_persister + .persist_new_channel(funding_txo, monitor, update_id)?; + if let Some(backup_path) = self.monitor_backup_path() { + let filename = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); + write_monitor_to_file(backup_path, filename, monitor) + .map_err(|_| ChannelMonitorUpdateErr::PermanentFailure)?; + } + Ok(()) + } + + fn update_persisted_channel( + &self, + funding_txo: OutPoint, + update: &Option, + monitor: &ChannelMonitor, + update_id: chainmonitor::MonitorUpdateId, + ) -> Result<(), ChannelMonitorUpdateErr> { + self.channels_persister + .update_persisted_channel(funding_txo, update, monitor, update_id)?; + if let Some(backup_path) = self.monitor_backup_path() { + let filename = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); + write_monitor_to_file(backup_path, filename, monitor) + .map_err(|_| ChannelMonitorUpdateErr::PermanentFailure)?; + } + Ok(()) + } +} + +#[async_trait] +impl LightningStorage for LightningFilesystemPersister { + type Error = std::io::Error; + + async fn init_fs(&self) -> Result<(), Self::Error> { + let path = self.main_path(); + let backup_path = self.backup_path(); + async_blocking(move || { + fs::create_dir_all(path.clone())?; + if let Some(path) = backup_path { + fs::create_dir_all(path.clone())?; + check_dir_operations(&path)?; + } + check_dir_operations(&path) + }) + .await + } + + async fn is_fs_initialized(&self) -> Result { + let dir_path = self.main_path(); + let backup_dir_path = self.backup_path(); + async_blocking(move || { + if !dir_path.exists() || backup_dir_path.as_ref().map(|path| !path.exists()).unwrap_or(false) { + Ok(false) + } else if !dir_path.is_dir() { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("{} is not a directory", dir_path.display()), + )) + } else if backup_dir_path.as_ref().map(|path| !path.is_dir()).unwrap_or(false) { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Backup path is not a directory", + )) + } else { + let check_backup_ops = if let Some(backup_path) = backup_dir_path { + check_dir_operations(&backup_path).is_ok() + } else { + true + }; + check_dir_operations(&dir_path).map(|_| check_backup_ops) + } + }) + .await + } + + async fn get_nodes_addresses(&self) -> Result { + let path = self.nodes_addresses_path(); + if !path.exists() { + return Ok(HashMap::new()); + } + async_blocking(move || { + let file = fs::File::open(path)?; + let reader = BufReader::new(file); + let nodes_addresses: HashMap = + serde_json::from_reader(reader).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + nodes_addresses + .iter() + .map(|(pubkey_str, addr)| { + let pubkey = PublicKey::from_str(pubkey_str) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + Ok((pubkey, *addr)) + }) + .collect() + }) + .await + } + + async fn save_nodes_addresses(&self, nodes_addresses: NodesAddressesMapShared) -> Result<(), Self::Error> { + let path = self.nodes_addresses_path(); + let backup_path = self.nodes_addresses_backup_path(); + async_blocking(move || { + let nodes_addresses: HashMap = nodes_addresses + .lock() + .iter() + .map(|(pubkey, addr)| (pubkey.to_string(), *addr)) + .collect(); + + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; + serde_json::to_writer(file, &nodes_addresses) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + + if let Some(path) = backup_path { + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; + serde_json::to_writer(file, &nodes_addresses) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + } + + Ok(()) + }) + .await + } + + async fn get_network_graph(&self, network: Network) -> Result { + let path = self.network_graph_path(); + if !path.exists() { + return Ok(NetworkGraph::new(genesis_block(network).header.block_hash())); + } + async_blocking(move || { + let file = fs::File::open(path)?; + common::log::info!("Reading the saved lightning network graph from file, this can take some time!"); + NetworkGraph::read(&mut BufReader::new(file)) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) + }) + .await + } + + async fn get_scorer(&self, network_graph: Arc) -> Result { + let path = self.scorer_path(); + if !path.exists() { + return Ok(Scorer::new(ProbabilisticScoringParameters::default(), network_graph)); + } + async_blocking(move || { + let file = fs::File::open(path)?; + Scorer::read( + &mut BufReader::new(file), + (ProbabilisticScoringParameters::default(), network_graph), + ) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) + }) + .await + } + + async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error> { + let path = self.scorer_path(); + async_blocking(move || { + let scorer = scorer.lock().unwrap(); + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; + scorer.write(&mut BufWriter::new(file)) + }) + .await + } +} diff --git a/mm2src/coins/lightning/ln_p2p.rs b/mm2src/coins/lightning/ln_p2p.rs index 00bd5cdd5b..cb4ab8c14a 100644 --- a/mm2src/coins/lightning/ln_p2p.rs +++ b/mm2src/coins/lightning/ln_p2p.rs @@ -7,7 +7,6 @@ use lightning::ln::msgs::NetAddress; use lightning::ln::peer_handler::{IgnoringMessageHandler, MessageHandler, SimpleArcPeerManager}; use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; use lightning_net_tokio::SocketDescriptor; -use lightning_persister::storage::NodesAddressesMapShared; use mm2_net::ip_addr::fetch_external_ip; use rand::RngCore; use secp256k1::SecretKey; diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index dfc3f24554..38535624a3 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -1,10 +1,8 @@ use super::*; -use crate::lightning::ln_errors::{FindWatchedOutputSpendError, GetHeaderError, GetTxError, SaveChannelClosingError, - SaveChannelClosingResult}; -use crate::utxo::rpc_clients::{electrum_script_hash, BestBlock as RpcBestBlock, BlockHashOrHeight, - ElectrumBlockHeader, ElectrumClient, ElectrumNonce, EstimateFeeMethod, - UtxoRpcClientEnum, UtxoRpcError}; -use crate::utxo::utxo_common; +use crate::lightning::ln_errors::{SaveChannelClosingError, SaveChannelClosingResult}; +use crate::utxo::rpc_clients::{BestBlock as RpcBestBlock, BlockHashOrHeight, ElectrumBlockHeader, ElectrumClient, + ElectrumNonce, EstimateFeeMethod, UtxoRpcClientEnum}; +use crate::utxo::spv::{ConfirmedTransactionInfo, SimplePaymentVerification}; use crate::utxo::utxo_standard::UtxoStandardCoin; use crate::{MarketCoinOps, MmCoin}; use bitcoin::blockdata::block::BlockHeader; @@ -14,15 +12,16 @@ use bitcoin::consensus::encode::{deserialize, serialize_hex}; use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid}; use bitcoin_hashes::{sha256d, Hash}; use common::executor::{spawn, Timer}; -use common::jsonrpc_client::JsonRpcErrorType; use common::log::{debug, error, info}; use futures::compat::Future01CompatExt; +use futures::future::join_all; use keys::hash::H256; use lightning::chain::{chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator}, Confirm, Filter, WatchedOutput}; -use rpc::v1::types::H256 as H256Json; +use rpc::v1::types::{Bytes as BytesJson, H256 as H256Json}; +use spv_validation::spv_proof::TRY_SPV_PROOF_INTERVAL; use std::cmp; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; const CHECK_FOR_NEW_BEST_BLOCK_INTERVAL: f64 = 60.; @@ -32,55 +31,8 @@ const TRY_LOOP_INTERVAL: f64 = 60.; #[inline] pub fn h256_json_from_txid(txid: Txid) -> H256Json { H256Json::from(txid.as_hash().into_inner()).reversed() } -struct TxWithBlockInfo { - tx: Transaction, - block_header: BlockHeader, - block_height: u64, -} - -async fn get_block_header(electrum_client: &ElectrumClient, height: u64) -> Result { - Ok(deserialize( - &electrum_client.blockchain_block_header(height).compat().await?, - )?) -} - -async fn find_watched_output_spend_with_header( - electrum_client: &ElectrumClient, - output: &WatchedOutput, -) -> Result, FindWatchedOutputSpendError> { - // from_block parameter is not used in find_output_spend for electrum clients - let utxo_client: UtxoRpcClientEnum = electrum_client.clone().into(); - let tx_hash = H256::from(output.outpoint.txid.as_hash().into_inner()); - let output_spend = match utxo_client - .find_output_spend( - tx_hash, - output.script_pubkey.as_ref(), - output.outpoint.index.into(), - BlockHashOrHeight::Hash(Default::default()), - ) - .compat() - .await - .map_err(FindWatchedOutputSpendError::RpcError)? - { - Some(output) => output, - None => return Ok(None), - }; - - let height = match output_spend.spent_in_block { - BlockHashOrHeight::Height(h) => h, - _ => return Err(FindWatchedOutputSpendError::HashNotHeight), - }; - let block_header = get_block_header(electrum_client, height as u64) - .await - .map_err(FindWatchedOutputSpendError::GetHeaderError)?; - let spending_tx = Transaction::try_from(output_spend.spending_tx)?; - - Ok(Some(TxWithBlockInfo { - tx: spending_tx, - block_header, - block_height: height as u64, - })) -} +#[inline] +pub fn h256_from_txid(txid: Txid) -> H256 { H256::from(txid.as_hash().into_inner()) } pub async fn get_best_header(best_header_listener: &ElectrumClient) -> EnableLightningResult { best_header_listener @@ -104,20 +56,8 @@ pub async fn update_best_block( return; }, }; - let prev_blockhash = match sha256d::Hash::from_slice(&h.prev_block_hash.0) { - Ok(h) => h, - Err(e) => { - error!("Error while parsing previous block hash for lightning node: {}", e); - return; - }, - }; - let merkle_root = match sha256d::Hash::from_slice(&h.merkle_root.0) { - Ok(h) => h, - Err(e) => { - error!("Error while parsing merkle root for lightning node: {}", e); - return; - }, - }; + let prev_blockhash = sha256d::Hash::from_inner(h.prev_block_hash.0); + let merkle_root = sha256d::Hash::from_inner(h.merkle_root.0); ( BlockHeader { version: h.version as i32, @@ -148,7 +88,7 @@ pub async fn update_best_block( pub async fn ln_best_block_update_loop( platform: Arc, - persister: Arc, + db: SqliteLightningDB, chain_monitor: Arc, channel_manager: Arc, best_header_listener: ElectrumClient, @@ -156,15 +96,18 @@ pub async fn ln_best_block_update_loop( ) { let mut current_best_block = best_block; loop { + // Transactions confirmations check can be done at every CHECK_FOR_NEW_BEST_BLOCK_INTERVAL instead of at every new block + // in case a transaction confirmation fails due to electrums being down. This way there will be no need to wait for a new + // block to confirm such transaction and causing delays. + platform + .process_txs_confirmations(&best_header_listener, &db, &chain_monitor, &channel_manager) + .await; let best_header = ok_or_continue_after_sleep!(get_best_header(&best_header_listener).await, TRY_LOOP_INTERVAL); if current_best_block != best_header.clone().into() { platform.update_best_block_height(best_header.block_height()); platform .process_txs_unconfirmations(&chain_monitor, &channel_manager) .await; - platform - .process_txs_confirmations(&best_header_listener, &persister, &chain_monitor, &channel_manager) - .await; current_best_block = best_header.clone().into(); update_best_block(&chain_monitor, &channel_manager, best_header).await; } @@ -172,22 +115,15 @@ pub async fn ln_best_block_update_loop( } } -struct ConfirmedTransactionInfo { - txid: Txid, - header: BlockHeader, - index: usize, - transaction: Transaction, - height: u32, -} - -impl ConfirmedTransactionInfo { - fn new(txid: Txid, header: BlockHeader, index: usize, transaction: Transaction, height: u32) -> Self { - ConfirmedTransactionInfo { - txid, - header, - index, - transaction, - height, +async fn get_funding_tx_bytes_loop(rpc_client: &UtxoRpcClientEnum, tx_hash: H256Json) -> BytesJson { + loop { + match rpc_client.get_transaction_bytes(&tx_hash).compat().await { + Ok(res) => break res, + Err(e) => { + error!("error {}", e); + Timer::sleep(TRY_LOOP_INTERVAL).await; + continue; + }, } } } @@ -202,7 +138,7 @@ pub struct Platform { /// estimate_fee_sat fails. pub default_fees_and_confirmations: PlatformCoinConfirmations, /// This cache stores the transactions that the LN node has interest in. - pub registered_txs: PaMutex>>, + pub registered_txs: PaMutex>, /// This cache stores the outputs that the LN node has interest in. pub registered_outputs: PaMutex>, /// This cache stores transactions to be broadcasted once the other node accepts the channel @@ -221,7 +157,7 @@ impl Platform { network, best_block_height: AtomicU64::new(0), default_fees_and_confirmations, - registered_txs: PaMutex::new(HashMap::new()), + registered_txs: PaMutex::new(HashSet::new()), registered_outputs: PaMutex::new(Vec::new()), unsigned_funding_txs: PaMutex::new(HashMap::new()), } @@ -238,12 +174,9 @@ impl Platform { #[inline] pub fn best_block_height(&self) -> u64 { self.best_block_height.load(AtomicOrdering::Relaxed) } - pub fn add_tx(&self, txid: Txid, script_pubkey: Script) { + pub fn add_tx(&self, txid: Txid) { let mut registered_txs = self.registered_txs.lock(); - registered_txs - .entry(txid) - .or_insert_with(HashSet::new) - .insert(script_pubkey); + registered_txs.insert(txid); } pub fn add_output(&self, output: WatchedOutput) { @@ -251,36 +184,12 @@ impl Platform { registered_outputs.push(output); } - async fn get_tx_if_onchain(&self, txid: Txid) -> Result, GetTxError> { - let txid = h256_json_from_txid(txid); - match self - .rpc_client() - .get_transaction_bytes(&txid) - .compat() - .await - .map_err(|e| e.into_inner()) - { - Ok(bytes) => Ok(Some(deserialize(&bytes.into_vec())?)), - Err(err) => { - if let UtxoRpcError::ResponseParseError(ref json_err) = err { - if let JsonRpcErrorType::Response(_, json) = &json_err.error { - if let Some(message) = json["message"].as_str() { - if message.contains(utxo_common::NO_TX_ERROR_CODE) { - return Ok(None); - } - } - } - } - Err(err.into()) - }, - } - } - async fn process_tx_for_unconfirmation(&self, txid: Txid, monitor: &T) where T: Confirm, { - match self.get_tx_if_onchain(txid).await { + let rpc_txid = h256_json_from_txid(txid); + match self.rpc_client().get_tx_if_onchain(&rpc_txid).await { Ok(Some(_)) => {}, Ok(None) => { info!( @@ -288,6 +197,10 @@ impl Platform { txid, ); monitor.transaction_unconfirmed(&txid); + // If a transaction is unconfirmed due to a block reorganization; LDK will rebroadcast it. + // In this case, this transaction needs to be added again to the registered transactions + // to start watching for it on the chain again. + self.add_tx(txid); }, Err(e) => error!( "Error while trying to check if the transaction {} is discarded or not :{:?}", @@ -312,48 +225,50 @@ impl Platform { async fn get_confirmed_registered_txs(&self, client: &ElectrumClient) -> Vec { let registered_txs = self.registered_txs.lock().clone(); - let mut confirmed_registered_txs = Vec::new(); - for (txid, scripts) in registered_txs { - if let Some(transaction) = - ok_or_continue_after_sleep!(self.get_tx_if_onchain(txid).await, TRY_LOOP_INTERVAL) - { - for (_, vout) in transaction.output.iter().enumerate() { - if scripts.contains(&vout.script_pubkey) { - let script_hash = hex::encode(electrum_script_hash(vout.script_pubkey.as_ref())); - let history = ok_or_retry_after_sleep!( - client.scripthash_get_history(&script_hash).compat().await, - TRY_LOOP_INTERVAL - ); - for item in history { - let rpc_txid = h256_json_from_txid(txid); - if item.tx_hash == rpc_txid && item.height > 0 { - let height = item.height as u64; - let header = - ok_or_retry_after_sleep!(get_block_header(client, height).await, TRY_LOOP_INTERVAL); - let index = ok_or_retry_after_sleep!( - client - .blockchain_transaction_get_merkle(rpc_txid, height) - .compat() - .await, - TRY_LOOP_INTERVAL - ) - .pos; - let confirmed_transaction_info = ConfirmedTransactionInfo::new( - txid, - header, - index, - transaction.clone(), - height as u32, - ); - confirmed_registered_txs.push(confirmed_transaction_info); - self.registered_txs.lock().remove(&txid); - } - } - } - } - } - } - confirmed_registered_txs + + let on_chain_txs_futs = registered_txs + .into_iter() + .map(|txid| async move { + let rpc_txid = h256_json_from_txid(txid); + self.rpc_client().get_tx_if_onchain(&rpc_txid).await + }) + .collect::>(); + let on_chain_txs = join_all(on_chain_txs_futs) + .await + .into_iter() + .filter_map(|maybe_tx| match maybe_tx { + Ok(maybe_tx) => maybe_tx, + Err(e) => { + error!( + "Error while trying to figure if transaction is on-chain or not: {:?}", + e + ); + None + }, + }); + + let confirmed_transactions_futs = on_chain_txs + .map(|transaction| async move { + client + .validate_spv_proof(&transaction, (now_ms() / 1000) + TRY_SPV_PROOF_INTERVAL) + .await + }) + .collect::>(); + join_all(confirmed_transactions_futs) + .await + .into_iter() + .filter_map(|confirmed_transaction| match confirmed_transaction { + Ok(confirmed_tx) => { + let txid = Txid::from_hash(confirmed_tx.tx.hash().reversed().to_sha256d()); + self.registered_txs.lock().remove(&txid); + Some(confirmed_tx) + }, + Err(e) => { + error!("Error verifying transaction: {:?}", e); + None + }, + }) + .collect() } async fn append_spent_registered_output_txs( @@ -361,47 +276,76 @@ impl Platform { transactions_to_confirm: &mut Vec, client: &ElectrumClient, ) { - let mut outputs_to_remove = Vec::new(); let registered_outputs = self.registered_outputs.lock().clone(); - for output in registered_outputs { - if let Some(tx_info) = ok_or_continue_after_sleep!( - find_watched_output_spend_with_header(client, &output).await, - TRY_LOOP_INTERVAL - ) { - if !transactions_to_confirm - .iter() - .any(|info| info.txid == tx_info.tx.txid()) - { - let rpc_txid = h256_json_from_txid(tx_info.tx.txid()); - let index = ok_or_retry_after_sleep!( - client - .blockchain_transaction_get_merkle(rpc_txid, tx_info.block_height) - .compat() - .await, - TRY_LOOP_INTERVAL + + let spent_outputs_info_fut = registered_outputs + .into_iter() + .map(|output| async move { + self.rpc_client() + .find_output_spend( + h256_from_txid(output.outpoint.txid), + output.script_pubkey.as_ref(), + output.outpoint.index.into(), + BlockHashOrHeight::Hash(Default::default()), ) - .pos; - let confirmed_transaction_info = ConfirmedTransactionInfo::new( - tx_info.tx.txid(), - tx_info.block_header, - index, - tx_info.tx, - tx_info.block_height as u32, - ); - transactions_to_confirm.push(confirmed_transaction_info); - } - outputs_to_remove.push(output); - } - } - self.registered_outputs - .lock() - .retain(|output| !outputs_to_remove.contains(output)); + .compat() + .await + }) + .collect::>(); + let mut spent_outputs_info = join_all(spent_outputs_info_fut) + .await + .into_iter() + .filter_map(|maybe_spent| match maybe_spent { + Ok(maybe_spent) => maybe_spent, + Err(e) => { + error!("Error while trying to figure if output is spent or not: {:?}", e); + None + }, + }) + .collect::>(); + spent_outputs_info.retain(|output| { + !transactions_to_confirm + .iter() + .any(|info| info.tx.hash() == output.spending_tx.hash()) + }); + + let confirmed_transactions_futs = spent_outputs_info + .into_iter() + .map(|output| async move { + client + .validate_spv_proof(&output.spending_tx, (now_ms() / 1000) + TRY_SPV_PROOF_INTERVAL) + .await + }) + .collect::>(); + let mut confirmed_transaction_info = join_all(confirmed_transactions_futs) + .await + .into_iter() + .filter_map(|confirmed_transaction| match confirmed_transaction { + Ok(confirmed_tx) => { + self.registered_outputs.lock().retain(|output| { + !confirmed_tx + .tx + .clone() + .inputs + .into_iter() + .any(|txin| txin.previous_output.hash == h256_from_txid(output.outpoint.txid)) + }); + Some(confirmed_tx) + }, + Err(e) => { + error!("Error verifying transaction: {:?}", e); + None + }, + }) + .collect(); + + transactions_to_confirm.append(&mut confirmed_transaction_info); } pub async fn process_txs_confirmations( &self, client: &ElectrumClient, - persister: &LightningPersister, + db: &SqliteLightningDB, chain_monitor: &ChainMonitor, channel_manager: &ChannelManager, ) { @@ -412,10 +356,10 @@ impl Platform { transactions_to_confirm.sort_by(|a, b| (a.height, a.index).cmp(&(b.height, b.index))); for confirmed_transaction_info in transactions_to_confirm { - let best_block_height = self.best_block_height(); - if let Err(e) = persister + let best_block_height = self.best_block_height() as i64; + if let Err(e) = db .update_funding_tx_block_height( - confirmed_transaction_info.transaction.txid().to_string(), + confirmed_transaction_info.tx.hash().reversed().to_string(), best_block_height, ) .await @@ -423,25 +367,25 @@ impl Platform { error!("Unable to update the funding tx block height in DB: {}", e); } channel_manager.transactions_confirmed( - &confirmed_transaction_info.header, + &confirmed_transaction_info.header.clone().into(), &[( - confirmed_transaction_info.index, - &confirmed_transaction_info.transaction, + confirmed_transaction_info.index as usize, + &confirmed_transaction_info.tx.clone().into(), )], - confirmed_transaction_info.height, + confirmed_transaction_info.height as u32, ); chain_monitor.transactions_confirmed( - &confirmed_transaction_info.header, + &confirmed_transaction_info.header.into(), &[( - confirmed_transaction_info.index, - &confirmed_transaction_info.transaction, + confirmed_transaction_info.index as usize, + &confirmed_transaction_info.tx.into(), )], - confirmed_transaction_info.height, + confirmed_transaction_info.height as u32, ); } } - pub async fn get_channel_closing_tx(&self, channel_details: SqlChannelDetails) -> SaveChannelClosingResult { + pub async fn get_channel_closing_tx(&self, channel_details: DBChannelDetails) -> SaveChannelClosingResult { let from_block = channel_details .funding_generated_in_block .ok_or_else(|| MmError::new(SaveChannelClosingError::BlockHeightNull))?; @@ -453,17 +397,14 @@ impl Platform { let tx_hash = H256Json::from_str(&tx_id).map_to_mm(|e| SaveChannelClosingError::FundingTxParseError(e.to_string()))?; - let funding_tx_bytes = ok_or_retry_after_sleep!( - self.rpc_client().get_transaction_bytes(&tx_hash).compat().await, - TRY_LOOP_INTERVAL - ); + let funding_tx_bytes = get_funding_tx_bytes_loop(self.rpc_client(), tx_hash).await; let closing_tx = self .coin .wait_for_tx_spend( &funding_tx_bytes.into_vec(), (now_ms() / 1000) + 3600, - from_block, + from_block.try_into()?, &None, ) .compat() @@ -521,6 +462,7 @@ impl BroadcasterInterface for Platform { spawn(async move { match fut.compat().await { Ok(id) => info!("Transaction broadcasted successfully: {:?} ", id), + // TODO: broadcast transaction through p2p network in case of error Err(e) => error!("Broadcast transaction {} failed: {}", txid, e), } }); @@ -530,12 +472,11 @@ impl BroadcasterInterface for Platform { impl Filter for Platform { // Watches for this transaction on-chain #[inline] - fn register_tx(&self, txid: &Txid, script_pubkey: &Script) { self.add_tx(*txid, script_pubkey.clone()); } + fn register_tx(&self, txid: &Txid, _script_pubkey: &Script) { self.add_tx(*txid); } // Watches for any transactions that spend this output on-chain fn register_output(&self, output: WatchedOutput) -> Option<(usize, Transaction)> { self.add_output(output.clone()); - let block_hash = match output.block_hash { Some(h) => H256Json::from(h.as_hash().into_inner()), None => return None, @@ -545,29 +486,20 @@ impl Filter for Platform { // the filter interface which includes register_output and register_tx should be used for electrum clients only, // this is the reason for initializing the filter as an option in the start_lightning function as it will be None // when implementing lightning for native clients - let output_spend_info = tokio::task::block_in_place(move || { - let delay = TRY_LOOP_INTERVAL as u64; - ok_or_retry_after_sleep_sync!( - self.rpc_client() - .find_output_spend( - H256::from(output.outpoint.txid.as_hash().into_inner()), - output.script_pubkey.as_ref(), - output.outpoint.index.into(), - BlockHashOrHeight::Hash(block_hash), - ) - .wait(), - delay - ) - }); + let output_spend_fut = self.rpc_client().find_output_spend( + h256_from_txid(output.outpoint.txid), + output.script_pubkey.as_ref(), + output.outpoint.index.into(), + BlockHashOrHeight::Hash(block_hash), + ); + let maybe_output_spend_res = + tokio::task::block_in_place(move || output_spend_fut.wait()).error_log_passthrough(); - if let Some(info) = output_spend_info { - match Transaction::try_from(info.spending_tx) { - Ok(tx) => Some((info.input_index, tx)), - Err(e) => { - error!("Can't convert transaction error: {}", e.to_string()); - return None; - }, - }; + if let Ok(Some(spent_output_info)) = maybe_output_spend_res { + match Transaction::try_from(spent_output_info.spending_tx) { + Ok(spending_tx) => return Some((spent_output_info.input_index, spending_tx)), + Err(e) => error!("Can't convert transaction error: {}", e.to_string()), + } } None diff --git a/mm2src/coins/lightning/ln_sql.rs b/mm2src/coins/lightning/ln_sql.rs new file mode 100644 index 0000000000..2cbeb98116 --- /dev/null +++ b/mm2src/coins/lightning/ln_sql.rs @@ -0,0 +1,1435 @@ +use crate::lightning::ln_db::{ChannelType, ChannelVisibility, ClosedChannelsFilter, DBChannelDetails, DBPaymentInfo, + DBPaymentsFilter, GetClosedChannelsResult, GetPaymentsResult, HTLCStatus, LightningDB, + PaymentType}; +use async_trait::async_trait; +use common::{async_blocking, PagingOptionsEnum}; +use db_common::sqlite::rusqlite::{Error as SqlError, Row, ToSql, NO_PARAMS}; +use db_common::sqlite::sql_builder::SqlBuilder; +use db_common::sqlite::{h256_option_slice_from_row, h256_slice_from_row, offset_by_id, query_single_row, + sql_text_conversion_err, string_from_row, validate_table_name, SqlNamedParams, + SqliteConnShared, CHECK_TABLE_EXISTS_SQL}; +use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; +use secp256k1::PublicKey; +use std::convert::TryInto; +use std::str::FromStr; + +fn channels_history_table(ticker: &str) -> String { ticker.to_owned() + "_channels_history" } + +fn payments_history_table(ticker: &str) -> String { ticker.to_owned() + "_payments_history" } + +fn create_channels_history_table_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + id INTEGER NOT NULL PRIMARY KEY, + rpc_id INTEGER NOT NULL UNIQUE, + channel_id VARCHAR(255) NOT NULL, + counterparty_node_id VARCHAR(255) NOT NULL, + funding_tx VARCHAR(255), + funding_value INTEGER, + funding_generated_in_block Integer, + closing_tx VARCHAR(255), + closure_reason TEXT, + claiming_tx VARCHAR(255), + claimed_balance REAL, + is_outbound INTEGER NOT NULL, + is_public INTEGER NOT NULL, + is_closed INTEGER NOT NULL, + created_at INTEGER NOT NULL, + closed_at INTEGER + );", + table_name + ); + + Ok(sql) +} + +fn create_payments_history_table_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + id INTEGER NOT NULL PRIMARY KEY, + payment_hash VARCHAR(255) NOT NULL UNIQUE, + destination VARCHAR(255), + description VARCHAR(641) NOT NULL, + preimage VARCHAR(255), + secret VARCHAR(255), + amount_msat INTEGER, + fee_paid_msat INTEGER, + is_outbound INTEGER NOT NULL, + status VARCHAR(255) NOT NULL, + created_at INTEGER NOT NULL, + last_updated INTEGER NOT NULL + );", + table_name + ); + + Ok(sql) +} + +fn insert_channel_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "INSERT INTO {} ( + rpc_id, + channel_id, + counterparty_node_id, + is_outbound, + is_public, + is_closed, + created_at + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7 + );", + table_name + ); + + Ok(sql) +} + +fn upsert_payment_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "INSERT OR REPLACE INTO {} ( + payment_hash, + destination, + description, + preimage, + secret, + amount_msat, + fee_paid_msat, + is_outbound, + status, + created_at, + last_updated + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 + );", + table_name + ); + + Ok(sql) +} + +fn select_channel_by_rpc_id_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "SELECT + rpc_id, + channel_id, + counterparty_node_id, + funding_tx, + funding_value, + funding_generated_in_block, + closing_tx, + closure_reason, + claiming_tx, + claimed_balance, + is_outbound, + is_public, + is_closed, + created_at, + closed_at + FROM + {} + WHERE + rpc_id=?1", + table_name + ); + + Ok(sql) +} + +fn select_payment_by_hash_sql(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "SELECT + payment_hash, + destination, + description, + preimage, + secret, + amount_msat, + fee_paid_msat, + status, + is_outbound, + created_at, + last_updated + FROM + {} + WHERE + payment_hash=?1;", + table_name + ); + + Ok(sql) +} + +fn channel_details_from_row(row: &Row<'_>) -> Result { + let channel_details = DBChannelDetails { + rpc_id: row.get(0)?, + channel_id: row.get(1)?, + counterparty_node_id: row.get(2)?, + funding_tx: row.get(3)?, + funding_value: row.get(4)?, + funding_generated_in_block: row.get(5)?, + closing_tx: row.get(6)?, + closure_reason: row.get(7)?, + claiming_tx: row.get(8)?, + claimed_balance: row.get(9)?, + is_outbound: row.get(10)?, + is_public: row.get(11)?, + is_closed: row.get(12)?, + created_at: row.get(13)?, + closed_at: row.get(14)?, + }; + Ok(channel_details) +} + +fn payment_info_from_row(row: &Row<'_>) -> Result { + let is_outbound = row.get::<_, bool>(8)?; + let payment_type = if is_outbound { + PaymentType::OutboundPayment { + destination: PublicKey::from_str(&row.get::<_, String>(1)?).map_err(|e| sql_text_conversion_err(1, e))?, + } + } else { + PaymentType::InboundPayment + }; + + let payment_info = DBPaymentInfo { + payment_hash: PaymentHash(h256_slice_from_row::(row, 0)?), + payment_type, + description: row.get(2)?, + preimage: h256_option_slice_from_row::(row, 3)?.map(PaymentPreimage), + secret: h256_option_slice_from_row::(row, 4)?.map(PaymentSecret), + amt_msat: row.get(5)?, + fee_paid_msat: row.get(6)?, + status: HTLCStatus::from_str(&row.get::<_, String>(7)?)?, + created_at: row.get(9)?, + last_updated: row.get(10)?, + }; + Ok(payment_info) +} + +fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!("SELECT IFNULL(MAX(rpc_id), 0) FROM {};", table_name); + + Ok(sql) +} + +fn update_funding_tx_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "UPDATE {} SET + funding_tx = ?1, + funding_value = ?2, + funding_generated_in_block = ?3 + WHERE + rpc_id = ?4;", + table_name + ); + + Ok(sql) +} + +fn update_funding_tx_block_height_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "UPDATE {} SET funding_generated_in_block = ?1 WHERE funding_tx = ?2;", + table_name + ); + + Ok(sql) +} + +fn update_channel_to_closed_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "UPDATE {} SET closure_reason = ?1, is_closed = ?2, closed_at = ?3 WHERE rpc_id = ?4;", + table_name + ); + + Ok(sql) +} + +fn update_closing_tx_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!("UPDATE {} SET closing_tx = ?1 WHERE rpc_id = ?2;", table_name); + + Ok(sql) +} + +fn get_channels_builder_preimage(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let mut sql_builder = SqlBuilder::select_from(table_name); + sql_builder.and_where("is_closed = 1"); + Ok(sql_builder) +} + +fn add_fields_to_get_channels_sql_builder(sql_builder: &mut SqlBuilder) { + sql_builder + .field("rpc_id") + .field("channel_id") + .field("counterparty_node_id") + .field("funding_tx") + .field("funding_value") + .field("funding_generated_in_block") + .field("closing_tx") + .field("closure_reason") + .field("claiming_tx") + .field("claimed_balance") + .field("is_outbound") + .field("is_public") + .field("is_closed") + .field("created_at") + .field("closed_at"); +} + +fn finalize_get_channels_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { + sql_builder.offset(offset); + sql_builder.limit(limit); + sql_builder.order_desc("closed_at"); +} + +fn apply_get_channels_filter<'a>( + builder: &mut SqlBuilder, + params: &mut SqlNamedParams<'a>, + filter: &'a ClosedChannelsFilter, +) { + if let Some(channel_id) = &filter.channel_id { + builder.and_where("channel_id = :channel_id"); + params.push((":channel_id", channel_id)); + } + + if let Some(counterparty_node_id) = &filter.counterparty_node_id { + builder.and_where("counterparty_node_id = :counterparty_node_id"); + params.push((":counterparty_node_id", counterparty_node_id)); + } + + if let Some(funding_tx) = &filter.funding_tx { + builder.and_where("funding_tx = :funding_tx"); + params.push((":funding_tx", funding_tx)); + } + + if let Some(from_funding_value) = &filter.from_funding_value { + builder.and_where("funding_value >= :from_funding_value"); + params.push((":from_funding_value", from_funding_value)); + } + + if let Some(to_funding_value) = &filter.to_funding_value { + builder.and_where("funding_value <= :to_funding_value"); + params.push((":to_funding_value", to_funding_value)); + } + + if let Some(closing_tx) = &filter.closing_tx { + builder.and_where("closing_tx = :closing_tx"); + params.push((":closing_tx", closing_tx)); + } + + if let Some(closure_reason) = &filter.closure_reason { + builder.and_where(format!("closure_reason LIKE '%{}%'", closure_reason)); + } + + if let Some(claiming_tx) = &filter.claiming_tx { + builder.and_where("claiming_tx = :claiming_tx"); + params.push((":claiming_tx", claiming_tx)); + } + + if let Some(from_claimed_balance) = &filter.from_claimed_balance { + builder.and_where("claimed_balance >= :from_claimed_balance"); + params.push((":from_claimed_balance", from_claimed_balance)); + } + + if let Some(to_claimed_balance) = &filter.to_claimed_balance { + builder.and_where("claimed_balance <= :to_claimed_balance"); + params.push((":to_claimed_balance", to_claimed_balance)); + } + + if let Some(channel_type) = &filter.channel_type { + let is_outbound = match channel_type { + ChannelType::Outbound => &true, + ChannelType::Inbound => &false, + }; + + builder.and_where("is_outbound = :is_outbound"); + params.push((":is_outbound", is_outbound)); + } + + if let Some(channel_visibility) = &filter.channel_visibility { + let is_public = match channel_visibility { + ChannelVisibility::Public => &true, + ChannelVisibility::Private => &false, + }; + + builder.and_where("is_public = :is_public"); + params.push((":is_public", is_public)); + } +} + +fn get_payments_builder_preimage(for_coin: &str) -> Result { + let table_name = payments_history_table(for_coin); + validate_table_name(&table_name)?; + + Ok(SqlBuilder::select_from(table_name)) +} + +fn finalize_get_payments_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { + sql_builder + .field("payment_hash") + .field("destination") + .field("description") + .field("preimage") + .field("secret") + .field("amount_msat") + .field("fee_paid_msat") + .field("status") + .field("is_outbound") + .field("created_at") + .field("last_updated"); + sql_builder.offset(offset); + sql_builder.limit(limit); + sql_builder.order_desc("last_updated"); +} + +fn apply_get_payments_filter<'a>( + builder: &mut SqlBuilder, + params: &mut SqlNamedParams<'a>, + filter: &'a DBPaymentsFilter, +) { + if let Some(dest) = &filter.destination { + builder.and_where("destination = :dest"); + params.push((":dest", dest)); + } + + if let Some(outbound) = &filter.is_outbound { + builder.and_where("is_outbound = :is_outbound"); + params.push((":is_outbound", outbound)); + } + + if let Some(description) = &filter.description { + builder.and_where(format!("description LIKE '%{}%'", description)); + } + + if let Some(status) = &filter.status { + builder.and_where("status = :status"); + params.push((":status", status)); + } + + if let Some(from_amount) = &filter.from_amount_msat { + builder.and_where("amount_msat >= :from_amount"); + params.push((":from_amount", from_amount)); + } + + if let Some(to_amount) = &filter.to_amount_msat { + builder.and_where("amount_msat <= :to_amount"); + params.push((":to_amount", to_amount)); + } + + if let Some(from_fee) = &filter.from_fee_paid_msat { + builder.and_where("fee_paid_msat >= :from_fee"); + params.push((":from_fee", from_fee)); + } + + if let Some(to_fee) = &filter.to_fee_paid_msat { + builder.and_where("fee_paid_msat <= :to_fee"); + params.push((":to_fee", to_fee)); + } + + if let Some(from_time) = &filter.from_timestamp { + builder.and_where("created_at >= :from_time"); + params.push((":from_time", from_time)); + } + + if let Some(to_time) = &filter.to_timestamp { + builder.and_where("created_at <= :to_time"); + params.push((":to_time", to_time)); + } +} + +fn update_claiming_tx_sql(for_coin: &str) -> Result { + let table_name = channels_history_table(for_coin); + validate_table_name(&table_name)?; + + let sql = format!( + "UPDATE {} SET claiming_tx = ?1, claimed_balance = ?2 WHERE closing_tx = ?3;", + table_name + ); + + Ok(sql) +} + +#[derive(Clone)] +pub struct SqliteLightningDB { + db_ticker: String, + sqlite_connection: SqliteConnShared, +} + +impl SqliteLightningDB { + pub fn new(ticker: String, sqlite_connection: SqliteConnShared) -> Self { + Self { + db_ticker: ticker.replace('-', "_"), + sqlite_connection, + } + } +} + +#[async_trait] +impl LightningDB for SqliteLightningDB { + type Error = SqlError; + + async fn init_db(&self) -> Result<(), Self::Error> { + let sqlite_connection = self.sqlite_connection.clone(); + + let sql_channels_history = create_channels_history_table_sql(self.db_ticker.as_str())?; + let sql_payments_history = create_payments_history_table_sql(self.db_ticker.as_str())?; + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + conn.execute(&sql_channels_history, NO_PARAMS).map(|_| ())?; + conn.execute(&sql_payments_history, NO_PARAMS).map(|_| ())?; + Ok(()) + }) + .await + } + + async fn is_db_initialized(&self) -> Result { + let channels_history_table = channels_history_table(self.db_ticker.as_str()); + validate_table_name(&channels_history_table)?; + let payments_history_table = payments_history_table(self.db_ticker.as_str()); + validate_table_name(&payments_history_table)?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let channels_history_initialized = + query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [channels_history_table], string_from_row)?; + let payments_history_initialized = + query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [payments_history_table], string_from_row)?; + Ok(channels_history_initialized.is_some() && payments_history_initialized.is_some()) + }) + .await + } + + async fn get_last_channel_rpc_id(&self) -> Result { + let sql = get_last_channel_rpc_id_sql(self.db_ticker.as_str())?; + let sqlite_connection = self.sqlite_connection.clone(); + + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + let count: u32 = conn.query_row(&sql, NO_PARAMS, |r| r.get(0))?; + Ok(count) + }) + .await + } + + async fn add_channel_to_db(&self, details: DBChannelDetails) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + let rpc_id = details.rpc_id as i64; + let created_at = details.created_at as i64; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [ + &rpc_id as &dyn ToSql, + &details.channel_id as &dyn ToSql, + &details.counterparty_node_id as &dyn ToSql, + &details.is_outbound as &dyn ToSql, + &details.is_public as &dyn ToSql, + &details.is_closed as &dyn ToSql, + &created_at as &dyn ToSql, + ]; + sql_transaction.execute(&insert_channel_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn add_funding_tx_to_db( + &self, + rpc_id: i64, + funding_tx: String, + funding_value: i64, + funding_generated_in_block: i64, + ) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [ + &funding_tx as &dyn ToSql, + &funding_value as &dyn ToSql, + &funding_generated_in_block as &dyn ToSql, + &rpc_id as &dyn ToSql, + ]; + sql_transaction.execute(&update_funding_tx_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: i64) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [&block_height as &dyn ToSql, &funding_tx as &dyn ToSql]; + sql_transaction.execute(&update_funding_tx_block_height_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn update_channel_to_closed( + &self, + rpc_id: i64, + closure_reason: String, + closed_at: i64, + ) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + let is_closed = true; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [ + &closure_reason as &dyn ToSql, + &is_closed as &dyn ToSql, + &closed_at as &dyn ToSql, + &rpc_id as &dyn ToSql, + ]; + sql_transaction.execute(&update_channel_to_closed_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error> { + let mut builder = get_channels_builder_preimage(self.db_ticker.as_str())?; + builder.and_where("funding_tx IS NOT NULL"); + builder.and_where("closing_tx IS NULL"); + add_fields_to_get_channels_sql_builder(&mut builder); + let sql = builder.sql().expect("valid sql"); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + + let mut stmt = conn.prepare(&sql)?; + let result = stmt + .query_map_named(&[], channel_details_from_row)? + .collect::>()?; + Ok(result) + }) + .await + } + + async fn add_closing_tx_to_db(&self, rpc_id: i64, closing_tx: String) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [&closing_tx as &dyn ToSql, &rpc_id as &dyn ToSql]; + sql_transaction.execute(&update_closing_tx_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn add_claiming_tx_to_db( + &self, + closing_tx: String, + claiming_tx: String, + claimed_balance: f64, + ) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [ + &claiming_tx as &dyn ToSql, + &claimed_balance as &dyn ToSql, + &closing_tx as &dyn ToSql, + ]; + sql_transaction.execute(&update_claiming_tx_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error> { + let params = [rpc_id.to_string()]; + let sql = select_channel_by_rpc_id_sql(self.db_ticker.as_str())?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + query_single_row(&conn, &sql, params, channel_details_from_row) + }) + .await + } + + async fn get_closed_channels_by_filter( + &self, + filter: Option, + paging: PagingOptionsEnum, + limit: usize, + ) -> Result { + let mut sql_builder = get_channels_builder_preimage(self.db_ticker.as_str())?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + + let mut total_builder = sql_builder.clone(); + total_builder.count("id"); + let total_sql = total_builder.sql().expect("valid sql"); + let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; + let total = total.try_into().expect("count should be always above zero"); + + let offset = match paging { + PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, + PagingOptionsEnum::FromId(rpc_id) => { + let params = [rpc_id as u32]; + let maybe_offset = + offset_by_id(&conn, &sql_builder, params, "rpc_id", "closed_at DESC", "rpc_id = ?1")?; + match maybe_offset { + Some(offset) => offset, + None => { + return Ok(GetClosedChannelsResult { + channels: vec![], + skipped: 0, + total, + }) + }, + } + }, + }; + + let mut params = vec![]; + if let Some(f) = &filter { + apply_get_channels_filter(&mut sql_builder, &mut params, f); + } + add_fields_to_get_channels_sql_builder(&mut sql_builder); + finalize_get_channels_sql_builder(&mut sql_builder, offset, limit); + + let sql = sql_builder.sql().expect("valid sql"); + let mut stmt = conn.prepare(&sql)?; + let channels = stmt + .query_map_named(params.as_slice(), channel_details_from_row)? + .collect::>()?; + let result = GetClosedChannelsResult { + channels, + skipped: offset, + total, + }; + Ok(result) + }) + .await + } + + async fn add_or_update_payment_in_db(&self, info: DBPaymentInfo) -> Result<(), Self::Error> { + let for_coin = self.db_ticker.clone(); + let payment_hash = hex::encode(info.payment_hash.0); + let (is_outbound, destination) = match info.payment_type { + PaymentType::OutboundPayment { destination } => (true, Some(destination.to_string())), + PaymentType::InboundPayment => (false, None), + }; + let preimage = info.preimage.map(|p| hex::encode(p.0)); + let secret = info.secret.map(|s| hex::encode(s.0)); + let status = info.status.to_string(); + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let params = [ + &payment_hash as &dyn ToSql, + &destination as &dyn ToSql, + &info.description as &dyn ToSql, + &preimage as &dyn ToSql, + &secret as &dyn ToSql, + &info.amt_msat as &dyn ToSql, + &info.fee_paid_msat as &dyn ToSql, + &is_outbound as &dyn ToSql, + &status as &dyn ToSql, + &info.created_at as &dyn ToSql, + &info.last_updated as &dyn ToSql, + ]; + let mut conn = sqlite_connection.lock().unwrap(); + let sql_transaction = conn.transaction()?; + sql_transaction.execute(&upsert_payment_sql(&for_coin)?, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error> { + let params = [hex::encode(hash.0)]; + let sql = select_payment_by_hash_sql(self.db_ticker.as_str())?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + query_single_row(&conn, &sql, params, payment_info_from_row) + }) + .await + } + + async fn get_payments_by_filter( + &self, + filter: Option, + paging: PagingOptionsEnum, + limit: usize, + ) -> Result { + let mut sql_builder = get_payments_builder_preimage(self.db_ticker.as_str())?; + + let sqlite_connection = self.sqlite_connection.clone(); + async_blocking(move || { + let conn = sqlite_connection.lock().unwrap(); + + let mut total_builder = sql_builder.clone(); + total_builder.count("id"); + let total_sql = total_builder.sql().expect("valid sql"); + let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; + let total = total.try_into().expect("count should be always above zero"); + + let offset = match paging { + PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, + PagingOptionsEnum::FromId(hash) => { + let hash_str = hex::encode(hash.0); + let params = [&hash_str]; + let maybe_offset = offset_by_id( + &conn, + &sql_builder, + params, + "payment_hash", + "last_updated DESC", + "payment_hash = ?1", + )?; + match maybe_offset { + Some(offset) => offset, + None => { + return Ok(GetPaymentsResult { + payments: vec![], + skipped: 0, + total, + }) + }, + } + }, + }; + + let mut params = vec![]; + if let Some(f) = &filter { + apply_get_payments_filter(&mut sql_builder, &mut params, f); + } + let params_as_trait: Vec<_> = params.iter().map(|(key, value)| (*key, value as &dyn ToSql)).collect(); + finalize_get_payments_sql_builder(&mut sql_builder, offset, limit); + + let sql = sql_builder.sql().expect("valid sql"); + let mut stmt = conn.prepare(&sql)?; + let payments = stmt + .query_map_named(params_as_trait.as_slice(), payment_info_from_row)? + .collect::>()?; + let result = GetPaymentsResult { + payments, + skipped: offset, + total, + }; + Ok(result) + }) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lightning::ln_db::DBChannelDetails; + use common::{block_on, now_ms}; + use db_common::sqlite::rusqlite::Connection; + use rand::distributions::Alphanumeric; + use rand::{Rng, RngCore}; + use secp256k1::{Secp256k1, SecretKey}; + use std::num::NonZeroUsize; + use std::sync::{Arc, Mutex}; + + fn generate_random_channels(num: u64) -> Vec { + let mut rng = rand::thread_rng(); + let mut channels = vec![]; + let s = Secp256k1::new(); + let mut bytes = [0; 32]; + for i in 0..num { + let details = DBChannelDetails { + rpc_id: (i + 1) as i64, + channel_id: { + rng.fill_bytes(&mut bytes); + hex::encode(bytes) + }, + counterparty_node_id: { + rng.fill_bytes(&mut bytes); + let secret = SecretKey::from_slice(&bytes).unwrap(); + let pubkey = PublicKey::from_secret_key(&s, &secret); + pubkey.to_string() + }, + funding_tx: { + rng.fill_bytes(&mut bytes); + Some(hex::encode(bytes)) + }, + funding_value: Some(rng.gen::()), + closing_tx: { + rng.fill_bytes(&mut bytes); + Some(hex::encode(bytes)) + }, + closure_reason: { + Some( + rng.sample_iter(&Alphanumeric) + .take(30) + .map(char::from) + .collect::(), + ) + }, + claiming_tx: { + rng.fill_bytes(&mut bytes); + Some(hex::encode(bytes)) + }, + claimed_balance: Some(rng.gen::()), + funding_generated_in_block: Some(rng.gen::()), + is_outbound: rng.gen::(), + is_public: rng.gen::(), + is_closed: rand::random(), + created_at: rng.gen::(), + closed_at: Some(rng.gen::()), + }; + channels.push(details); + } + channels + } + + fn generate_random_payments(num: u64) -> Vec { + let mut rng = rand::thread_rng(); + let mut payments = vec![]; + let s = Secp256k1::new(); + let mut bytes = [0; 32]; + for _ in 0..num { + let payment_type = if rng.gen::() { + rng.fill_bytes(&mut bytes); + let secret = SecretKey::from_slice(&bytes).unwrap(); + PaymentType::OutboundPayment { + destination: PublicKey::from_secret_key(&s, &secret), + } + } else { + PaymentType::InboundPayment + }; + let status_rng: u8 = rng.gen(); + let status = if status_rng % 3 == 0 { + HTLCStatus::Succeeded + } else if status_rng % 3 == 1 { + HTLCStatus::Pending + } else { + HTLCStatus::Failed + }; + let description: String = rng.sample_iter(&Alphanumeric).take(30).map(char::from).collect(); + let info = DBPaymentInfo { + payment_hash: { + rng.fill_bytes(&mut bytes); + PaymentHash(bytes) + }, + payment_type, + description, + preimage: { + rng.fill_bytes(&mut bytes); + Some(PaymentPreimage(bytes)) + }, + secret: { + rng.fill_bytes(&mut bytes); + Some(PaymentSecret(bytes)) + }, + amt_msat: Some(rng.gen::()), + fee_paid_msat: Some(rng.gen::()), + status, + created_at: rng.gen::(), + last_updated: rng.gen::(), + }; + payments.push(info); + } + payments + } + + #[test] + fn test_init_sql_collection() { + let db = SqliteLightningDB::new( + "init_sql_collection".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + let initialized = block_on(db.is_db_initialized()).unwrap(); + assert!(!initialized); + + block_on(db.init_db()).unwrap(); + // repetitive init must not fail + block_on(db.init_db()).unwrap(); + + let initialized = block_on(db.is_db_initialized()).unwrap(); + assert!(initialized); + } + + #[test] + fn test_add_get_channel_sql() { + let db = SqliteLightningDB::new( + "add_get_channel".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(db.init_db()).unwrap(); + + let last_channel_rpc_id = block_on(db.get_last_channel_rpc_id()).unwrap(); + assert_eq!(last_channel_rpc_id, 0); + + let channel = block_on(db.get_channel_from_db(1)).unwrap(); + assert!(channel.is_none()); + + let mut expected_channel_details = DBChannelDetails::new( + 1, + [0; 32], + PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), + true, + true, + ); + block_on(db.add_channel_to_db(expected_channel_details.clone())).unwrap(); + let last_channel_rpc_id = block_on(db.get_last_channel_rpc_id()).unwrap(); + assert_eq!(last_channel_rpc_id, 1); + + let actual_channel_details = block_on(db.get_channel_from_db(1)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + // must fail because we are adding channel with the same rpc_id + block_on(db.add_channel_to_db(expected_channel_details.clone())).unwrap_err(); + assert_eq!(last_channel_rpc_id, 1); + + expected_channel_details.rpc_id = 2; + block_on(db.add_channel_to_db(expected_channel_details.clone())).unwrap(); + let last_channel_rpc_id = block_on(db.get_last_channel_rpc_id()).unwrap(); + assert_eq!(last_channel_rpc_id, 2); + + block_on(db.add_funding_tx_to_db( + 2, + "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), + 3000, + 50000, + )) + .unwrap(); + expected_channel_details.funding_tx = + Some("9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into()); + expected_channel_details.funding_value = Some(3000); + expected_channel_details.funding_generated_in_block = Some(50000); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + block_on(db.update_funding_tx_block_height( + "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), + 50001, + )) + .unwrap(); + expected_channel_details.funding_generated_in_block = Some(50001); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + let current_time = (now_ms() / 1000) as i64; + block_on(db.update_channel_to_closed(2, "the channel was cooperatively closed".into(), current_time)).unwrap(); + expected_channel_details.closure_reason = Some("the channel was cooperatively closed".into()); + expected_channel_details.is_closed = true; + expected_channel_details.closed_at = Some(current_time); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + let closed_channels = + block_on(db.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); + assert_eq!(closed_channels.channels.len(), 1); + assert_eq!(expected_channel_details, closed_channels.channels[0]); + + block_on(db.update_channel_to_closed( + 1, + "the channel was cooperatively closed".into(), + (now_ms() / 1000) as i64, + )) + .unwrap(); + let closed_channels = + block_on(db.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); + assert_eq!(closed_channels.channels.len(), 2); + + let actual_channels = block_on(db.get_closed_channels_with_no_closing_tx()).unwrap(); + assert_eq!(actual_channels.len(), 1); + + block_on(db.add_closing_tx_to_db( + 2, + "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), + )) + .unwrap(); + expected_channel_details.closing_tx = + Some("5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into()); + + let actual_channels = block_on(db.get_closed_channels_with_no_closing_tx()).unwrap(); + assert!(actual_channels.is_empty()); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + + block_on(db.add_claiming_tx_to_db( + "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), + "97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into(), + 2000.333333, + )) + .unwrap(); + expected_channel_details.claiming_tx = + Some("97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into()); + expected_channel_details.claimed_balance = Some(2000.333333); + + let actual_channel_details = block_on(db.get_channel_from_db(2)).unwrap().unwrap(); + assert_eq!(expected_channel_details, actual_channel_details); + } + + #[test] + fn test_add_get_payment_sql() { + let db = SqliteLightningDB::new( + "add_get_payment".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(db.init_db()).unwrap(); + + let payment = block_on(db.get_payment_from_db(PaymentHash([0; 32]))).unwrap(); + assert!(payment.is_none()); + + let mut expected_payment_info = DBPaymentInfo { + payment_hash: PaymentHash([0; 32]), + payment_type: PaymentType::InboundPayment, + description: "test payment".into(), + preimage: Some(PaymentPreimage([2; 32])), + secret: Some(PaymentSecret([3; 32])), + amt_msat: Some(2000), + fee_paid_msat: Some(100), + status: HTLCStatus::Failed, + created_at: (now_ms() / 1000) as i64, + last_updated: (now_ms() / 1000) as i64, + }; + block_on(db.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); + + let actual_payment_info = block_on(db.get_payment_from_db(PaymentHash([0; 32]))).unwrap().unwrap(); + assert_eq!(expected_payment_info, actual_payment_info); + + expected_payment_info.payment_hash = PaymentHash([1; 32]); + expected_payment_info.payment_type = PaymentType::OutboundPayment { + destination: PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9") + .unwrap(), + }; + expected_payment_info.secret = None; + expected_payment_info.amt_msat = None; + expected_payment_info.status = HTLCStatus::Succeeded; + expected_payment_info.last_updated = (now_ms() / 1000) as i64; + block_on(db.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); + + let actual_payment_info = block_on(db.get_payment_from_db(PaymentHash([1; 32]))).unwrap().unwrap(); + assert_eq!(expected_payment_info, actual_payment_info); + } + + #[test] + fn test_get_payments_by_filter() { + let db = SqliteLightningDB::new( + "test_get_payments_by_filter".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(db.init_db()).unwrap(); + + let mut payments = generate_random_payments(100); + + for payment in payments.clone() { + block_on(db.add_or_update_payment_in_db(payment)).unwrap(); + } + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 4; + + let result = block_on(db.get_payments_by_filter(None, paging, limit)).unwrap(); + + payments.sort_by(|a, b| b.last_updated.cmp(&a.last_updated)); + let expected_payments = &payments[..4].to_vec(); + let actual_payments = &result.payments; + + assert_eq!(0, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_payments, actual_payments); + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); + let limit = 5; + + let result = block_on(db.get_payments_by_filter(None, paging, limit)).unwrap(); + + let expected_payments = &payments[5..10].to_vec(); + let actual_payments = &result.payments; + + assert_eq!(5, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_payments, actual_payments); + + let from_payment_hash = payments[20].payment_hash; + let paging = PagingOptionsEnum::FromId(from_payment_hash); + let limit = 3; + + let result = block_on(db.get_payments_by_filter(None, paging, limit)).unwrap(); + + let expected_payments = &payments[21..24].to_vec(); + let actual_payments = &result.payments; + + assert_eq!(expected_payments, actual_payments); + + let mut filter = DBPaymentsFilter { + is_outbound: Some(false), + destination: None, + description: None, + status: None, + from_amount_msat: None, + to_amount_msat: None, + from_fee_paid_msat: None, + to_fee_paid_msat: None, + from_timestamp: None, + to_timestamp: None, + }; + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 10; + + let result = block_on(db.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_payments_vec: Vec = payments + .iter() + .map(|p| p.clone()) + .filter(|p| p.payment_type == PaymentType::InboundPayment) + .collect(); + let expected_payments = if expected_payments_vec.len() > 10 { + expected_payments_vec[..10].to_vec() + } else { + expected_payments_vec.clone() + }; + let actual_payments = result.payments; + + assert_eq!(expected_payments, actual_payments); + + filter.status = Some(HTLCStatus::Succeeded.to_string()); + let result = block_on(db.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_payments_vec: Vec = expected_payments_vec + .iter() + .map(|p| p.clone()) + .filter(|p| p.status == HTLCStatus::Succeeded) + .collect(); + let expected_payments = if expected_payments_vec.len() > 10 { + expected_payments_vec[..10].to_vec() + } else { + expected_payments_vec + }; + let actual_payments = result.payments; + + assert_eq!(expected_payments, actual_payments); + + let description = &payments[42].description; + let substr = &description[5..10]; + filter.is_outbound = None; + filter.destination = None; + filter.status = None; + filter.description = Some(substr.to_string()); + let result = block_on(db.get_payments_by_filter(Some(filter), paging, limit)).unwrap(); + let expected_payments_vec: Vec = payments + .iter() + .map(|p| p.clone()) + .filter(|p| p.description.contains(&substr)) + .collect(); + let expected_payments = if expected_payments_vec.len() > 10 { + expected_payments_vec[..10].to_vec() + } else { + expected_payments_vec.clone() + }; + let actual_payments = result.payments; + + assert_eq!(expected_payments, actual_payments); + } + + #[test] + fn test_get_channels_by_filter() { + let db = SqliteLightningDB::new( + "test_get_channels_by_filter".into(), + Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), + ); + + block_on(db.init_db()).unwrap(); + + let channels = generate_random_channels(100); + + for channel in channels { + block_on(db.add_channel_to_db(channel.clone())).unwrap(); + block_on(db.add_funding_tx_to_db( + channel.rpc_id, + channel.funding_tx.unwrap(), + channel.funding_value.unwrap(), + channel.funding_generated_in_block.unwrap(), + )) + .unwrap(); + block_on(db.update_channel_to_closed(channel.rpc_id, channel.closure_reason.unwrap(), 1655806080)).unwrap(); + block_on(db.add_closing_tx_to_db(channel.rpc_id, channel.closing_tx.clone().unwrap())).unwrap(); + block_on(db.add_claiming_tx_to_db( + channel.closing_tx.unwrap(), + channel.claiming_tx.unwrap(), + channel.claimed_balance.unwrap(), + )) + .unwrap(); + } + + // get all channels from SQL since updated_at changed from channels generated by generate_random_channels + let channels = block_on(db.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 100)) + .unwrap() + .channels; + assert_eq!(100, channels.len()); + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 4; + + let result = block_on(db.get_closed_channels_by_filter(None, paging, limit)).unwrap(); + + let expected_channels = &channels[..4].to_vec(); + let actual_channels = &result.channels; + + assert_eq!(0, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_channels, actual_channels); + + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); + let limit = 5; + + let result = block_on(db.get_closed_channels_by_filter(None, paging, limit)).unwrap(); + + let expected_channels = &channels[5..10].to_vec(); + let actual_channels = &result.channels; + + assert_eq!(5, result.skipped); + assert_eq!(100, result.total); + assert_eq!(expected_channels, actual_channels); + + let from_rpc_id = 20; + let paging = PagingOptionsEnum::FromId(from_rpc_id); + let limit = 3; + + let result = block_on(db.get_closed_channels_by_filter(None, paging, limit)).unwrap(); + + let expected_channels = channels[20..23].to_vec(); + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + + let mut filter = ClosedChannelsFilter { + channel_id: None, + counterparty_node_id: None, + funding_tx: None, + from_funding_value: None, + to_funding_value: None, + closing_tx: None, + closure_reason: None, + claiming_tx: None, + from_claimed_balance: None, + to_claimed_balance: None, + channel_type: Some(ChannelType::Outbound), + channel_visibility: None, + }; + let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); + let limit = 10; + + let result = block_on(db.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_channels_vec: Vec = channels + .iter() + .map(|chan| chan.clone()) + .filter(|chan| chan.is_outbound) + .collect(); + let expected_channels = if expected_channels_vec.len() > 10 { + expected_channels_vec[..10].to_vec() + } else { + expected_channels_vec.clone() + }; + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + + filter.channel_visibility = Some(ChannelVisibility::Public); + let result = block_on(db.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); + let expected_channels_vec: Vec = expected_channels_vec + .iter() + .map(|chan| chan.clone()) + .filter(|chan| chan.is_public) + .collect(); + let expected_channels = if expected_channels_vec.len() > 10 { + expected_channels_vec[..10].to_vec() + } else { + expected_channels_vec + }; + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + + let channel_id = channels[42].channel_id.clone(); + filter.channel_type = None; + filter.channel_visibility = None; + filter.channel_id = Some(channel_id.clone()); + let result = block_on(db.get_closed_channels_by_filter(Some(filter), paging, limit)).unwrap(); + let expected_channels_vec: Vec = channels + .iter() + .map(|chan| chan.clone()) + .filter(|chan| chan.channel_id == channel_id) + .collect(); + let expected_channels = if expected_channels_vec.len() > 10 { + expected_channels_vec[..10].to_vec() + } else { + expected_channels_vec.clone() + }; + let actual_channels = result.channels; + + assert_eq!(expected_channels, actual_channels); + } +} diff --git a/mm2src/coins/lightning/ln_storage.rs b/mm2src/coins/lightning/ln_storage.rs new file mode 100644 index 0000000000..bd44fdc0e1 --- /dev/null +++ b/mm2src/coins/lightning/ln_storage.rs @@ -0,0 +1,33 @@ +use async_trait::async_trait; +use bitcoin::Network; +use lightning::routing::network_graph::NetworkGraph; +use lightning::routing::scoring::ProbabilisticScorer; +use parking_lot::Mutex as PaMutex; +use secp256k1::PublicKey; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; + +pub type NodesAddressesMap = HashMap; +pub type NodesAddressesMapShared = Arc>; +pub type Scorer = ProbabilisticScorer>; + +#[async_trait] +pub trait LightningStorage { + type Error; + + /// Initializes dirs/collection/tables in storage for a specified coin + async fn init_fs(&self) -> Result<(), Self::Error>; + + async fn is_fs_initialized(&self) -> Result; + + async fn get_nodes_addresses(&self) -> Result, Self::Error>; + + async fn save_nodes_addresses(&self, nodes_addresses: NodesAddressesMapShared) -> Result<(), Self::Error>; + + async fn get_network_graph(&self, network: Network) -> Result; + + async fn get_scorer(&self, network_graph: Arc) -> Result; + + async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error>; +} diff --git a/mm2src/coins/lightning/ln_utils.rs b/mm2src/coins/lightning/ln_utils.rs index 05e4fd4f63..30b3e3c2de 100644 --- a/mm2src/coins/lightning/ln_utils.rs +++ b/mm2src/coins/lightning/ln_utils.rs @@ -1,5 +1,9 @@ use super::*; +use crate::lightning::ln_db::LightningDB; +use crate::lightning::ln_filesystem_persister::LightningPersisterShared; use crate::lightning::ln_platform::{get_best_header, ln_best_block_update_loop, update_best_block}; +use crate::lightning::ln_sql::SqliteLightningDB; +use crate::lightning::ln_storage::{LightningStorage, NodesAddressesMap, Scorer}; use crate::utxo::rpc_clients::BestBlock as RpcBestBlock; use bitcoin::hash_types::BlockHash; use bitcoin_hashes::{sha256d, Hash}; @@ -10,18 +14,14 @@ use lightning::chain::keysinterface::{InMemorySigner, KeysManager}; use lightning::chain::{chainmonitor, BestBlock, Watch}; use lightning::ln::channelmanager; use lightning::ln::channelmanager::{ChainParameters, ChannelManagerReadArgs, SimpleArcChannelManager}; -use lightning::routing::network_graph::NetworkGraph; use lightning::util::config::UserConfig; use lightning::util::ser::ReadableArgs; -use lightning_persister::storage::{DbStorage, FileSystemStorage, NodesAddressesMap, Scorer}; -use lightning_persister::LightningPersister; use mm2_core::mm_ctx::MmArc; use std::fs::File; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::SystemTime; -const NETWORK_GRAPH_PERSIST_INTERVAL: u64 = 600; const SCORER_PERSIST_INTERVAL: u64 = 600; pub type ChainMonitor = chainmonitor::ChainMonitor< @@ -30,7 +30,7 @@ pub type ChainMonitor = chainmonitor::ChainMonitor< Arc, Arc, Arc, - Arc, + LightningPersisterShared, >; pub type ChannelManager = SimpleArcChannelManager; @@ -50,54 +50,39 @@ fn ln_data_backup_dir(ctx: &MmArc, path: Option, ticker: &str) -> Option pub async fn init_persister( ctx: &MmArc, - platform: Arc, ticker: String, backup_path: Option, -) -> EnableLightningResult> { +) -> EnableLightningResult { let ln_data_dir = ln_data_dir(ctx, &ticker); let ln_data_backup_dir = ln_data_backup_dir(ctx, backup_path, &ticker); - let persister = Arc::new(LightningPersister::new( - ticker.replace('-', "_"), + let persister = LightningPersisterShared(Arc::new(LightningFilesystemPersister::new( ln_data_dir, ln_data_backup_dir, + ))); + + let is_initialized = persister.is_fs_initialized().await?; + if !is_initialized { + persister.init_fs().await?; + } + + Ok(persister) +} + +pub async fn init_db(ctx: &MmArc, ticker: String) -> EnableLightningResult { + let db = SqliteLightningDB::new( + ticker, ctx.sqlite_connection .ok_or(MmError::new(EnableLightningError::DbError( "sqlite_connection is not initialized".into(), )))? .clone(), - )); - let is_initialized = persister.is_fs_initialized().await?; - if !is_initialized { - persister.init_fs().await?; - } - let is_db_initialized = persister.is_db_initialized().await?; - if !is_db_initialized { - persister.init_db().await?; - } + ); - let closed_channels_without_closing_tx = persister.get_closed_channels_with_no_closing_tx().await?; - for channel_details in closed_channels_without_closing_tx { - let platform = platform.clone(); - let persister = persister.clone(); - let user_channel_id = channel_details.rpc_id; - spawn(async move { - if let Ok(closing_tx_hash) = platform - .get_channel_closing_tx(channel_details) - .await - .error_log_passthrough() - { - if let Err(e) = persister.add_closing_tx_to_db(user_channel_id, closing_tx_hash).await { - log::error!( - "Unable to update channel {} closing details in DB: {}", - user_channel_id, - e - ); - } - } - }); + if !db.is_db_initialized().await? { + db.init_db().await?; } - Ok(persister) + Ok(db) } pub fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> { @@ -113,7 +98,8 @@ pub fn init_keys_manager(ctx: &MmArc) -> EnableLightningResult> pub async fn init_channel_manager( platform: Arc, logger: Arc, - persister: Arc, + persister: LightningPersisterShared, + db: SqliteLightningDB, keys_manager: Arc, user_config: UserConfig, ) -> EnableLightningResult<(Arc, Arc)> { @@ -135,6 +121,7 @@ pub async fn init_channel_manager( // Read ChannelMonitor state from disk, important for lightning node is restarting and has at least 1 channel let mut channelmonitors = persister + .channels_persister() .read_channelmonitors(keys_manager.clone()) .map_to_mm(|e| EnableLightningError::IOError(e.to_string()))?; @@ -155,9 +142,7 @@ pub async fn init_channel_manager( let best_header = get_best_header(&rpc_client).await?; platform.update_best_block_height(best_header.block_height()); let best_block = RpcBestBlock::from(best_header.clone()); - let best_block_hash = BlockHash::from_hash( - sha256d::Hash::from_slice(&best_block.hash.0).map_to_mm(|e| EnableLightningError::HashError(e.to_string()))?, - ); + let best_block_hash = BlockHash::from_hash(sha256d::Hash::from_inner(best_block.hash.0)); let (channel_manager_blockhash, channel_manager) = { if let Ok(mut f) = File::open(persister.manager_path()) { let mut channel_monitor_mut_references = Vec::new(); @@ -198,13 +183,13 @@ pub async fn init_channel_manager( let channel_manager: Arc = Arc::new(channel_manager); // Sync ChannelMonitors and ChannelManager to chain tip if the node is restarting and has open channels + platform + .process_txs_confirmations(&rpc_client, &db, &chain_monitor, &channel_manager) + .await; if channel_manager_blockhash != best_block_hash { platform .process_txs_unconfirmations(&chain_monitor, &channel_manager) .await; - platform - .process_txs_confirmations(&rpc_client, &persister, &chain_monitor, &channel_manager) - .await; update_best_block(&chain_monitor, &channel_manager, best_header).await; } @@ -218,9 +203,8 @@ pub async fn init_channel_manager( // Update best block whenever there's a new chain tip or a block has been newly disconnected spawn(ln_best_block_update_loop( - // It's safe to use unwrap here for now until implementing Native Client for Lightning platform, - persister.clone(), + db, chain_monitor.clone(), channel_manager.clone(), rpc_client.clone(), @@ -230,19 +214,7 @@ pub async fn init_channel_manager( Ok((chain_monitor, channel_manager)) } -pub async fn persist_network_graph_loop(persister: Arc, network_graph: Arc) { - loop { - if let Err(e) = persister.save_network_graph(network_graph.clone()).await { - log::warn!( - "Failed to persist network graph error: {}, please check disk space and permissions", - e - ); - } - Timer::sleep(NETWORK_GRAPH_PERSIST_INTERVAL as f64).await; - } -} - -pub async fn persist_scorer_loop(persister: Arc, scorer: Arc>) { +pub async fn persist_scorer_loop(persister: LightningPersisterShared, scorer: Arc>) { loop { if let Err(e) = persister.save_scorer(scorer.clone()).await { log::warn!( @@ -255,7 +227,7 @@ pub async fn persist_scorer_loop(persister: Arc, scorer: Arc } pub async fn get_open_channels_nodes_addresses( - persister: Arc, + persister: LightningPersisterShared, channel_manager: Arc, ) -> EnableLightningResult { let channels = channel_manager.list_channels(); diff --git a/mm2src/coins/lightning_background_processor/Cargo.toml b/mm2src/coins/lightning_background_processor/Cargo.toml deleted file mode 100644 index 5710dcfc2c..0000000000 --- a/mm2src/coins/lightning_background_processor/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "lightning-background-processor" -version = "0.0.106" -authors = ["Valentine Wallace "] -license = "MIT OR Apache-2.0" -repository = "http://github.com/lightningdevkit/rust-lightning" -description = """ -Utilities to perform required background tasks for Rust Lightning. -""" -edition = "2018" - -[dependencies] -bitcoin = "0.27.1" -lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106", features = ["std"] } - -[dev-dependencies] -db_common = { path = "../../db_common" } -lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106", features = ["_test_utils"] } -lightning-invoice = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } -lightning-persister = { version = "0.0.106", path = "../lightning_persister" } diff --git a/mm2src/coins/lightning_background_processor/src/lib.rs b/mm2src/coins/lightning_background_processor/src/lib.rs deleted file mode 100644 index 4ca2fe9ad4..0000000000 --- a/mm2src/coins/lightning_background_processor/src/lib.rs +++ /dev/null @@ -1,950 +0,0 @@ -//! Utilities that take care of tasks that (1) need to happen periodically to keep Rust-Lightning -//! running properly, and (2) either can or should be run in the background. See docs for -//! [`BackgroundProcessor`] for more details on the nitty-gritty. - -#[macro_use] extern crate lightning; - -use lightning::chain; -use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; -use lightning::chain::chainmonitor::{ChainMonitor, Persist}; -use lightning::chain::keysinterface::{KeysInterface, Sign}; -use lightning::ln::channelmanager::ChannelManager; -use lightning::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler}; -use lightning::ln::peer_handler::{CustomMessageHandler, PeerManager, SocketDescriptor}; -use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; -use lightning::util::events::{Event, EventHandler, EventsProvider}; -use lightning::util::logger::Logger; -use std::ops::Deref; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::thread; -use std::thread::JoinHandle; -use std::time::{Duration, Instant}; - -/// `BackgroundProcessor` takes care of tasks that (1) need to happen periodically to keep -/// Rust-Lightning running properly, and (2) either can or should be run in the background. Its -/// responsibilities are: -/// * Processing [`Event`]s with a user-provided [`EventHandler`]. -/// * Monitoring whether the [`ChannelManager`] needs to be re-persisted to disk, and if so, -/// writing it to disk/backups by invoking the callback given to it at startup. -/// [`ChannelManager`] persistence should be done in the background. -/// * Calling [`ChannelManager::timer_tick_occurred`] and [`PeerManager::timer_tick_occurred`] -/// at the appropriate intervals. -/// * Calling [`NetworkGraph::remove_stale_channels`] (if a [`NetGraphMsgHandler`] is provided to -/// [`BackgroundProcessor::start`]). -/// -/// It will also call [`PeerManager::process_events`] periodically though this shouldn't be relied -/// upon as doing so may result in high latency. -/// -/// # Note -/// -/// If [`ChannelManager`] persistence fails and the persisted manager becomes out-of-date, then -/// there is a risk of channels force-closing on startup when the manager realizes it's outdated. -/// However, as long as [`ChannelMonitor`] backups are sound, no funds besides those used for -/// unilateral chain closure fees are at risk. -/// -/// [`ChannelMonitor`]: lightning::chain::channelmonitor::ChannelMonitor -/// [`Event`]: lightning::util::events::Event -#[must_use = "BackgroundProcessor will immediately stop on drop. It should be stored until shutdown."] -pub struct BackgroundProcessor { - stop_thread: Arc, - thread_handle: Option>>, -} - -#[cfg(not(test))] -const FRESHNESS_TIMER: u64 = 60; -#[cfg(test)] -const FRESHNESS_TIMER: u64 = 1; - -#[cfg(all(not(test), not(debug_assertions)))] -const PING_TIMER: u64 = 10; -/// Signature operations take a lot longer without compiler optimisations. -/// Increasing the ping timer allows for this but slower devices will be disconnected if the -/// timeout is reached. -#[cfg(all(not(test), debug_assertions))] -const PING_TIMER: u64 = 30; -#[cfg(test)] -const PING_TIMER: u64 = 1; - -/// Prune the network graph of stale entries hourly. -const NETWORK_PRUNE_TIMER: u64 = 60 * 60; - -/// Trait which handles persisting a [`ChannelManager`] to disk. -/// -/// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager -pub trait ChannelManagerPersister -where - M::Target: 'static + chain::Watch, - T::Target: 'static + BroadcasterInterface, - K::Target: 'static + KeysInterface, - F::Target: 'static + FeeEstimator, - L::Target: 'static + Logger, -{ - /// Persist the given [`ChannelManager`] to disk, returning an error if persistence failed - /// (which will cause the [`BackgroundProcessor`] which called this method to exit. - /// - /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager - fn persist_manager(&self, channel_manager: &ChannelManager) -> Result<(), std::io::Error>; -} - -impl ChannelManagerPersister - for Fun -where - M::Target: 'static + chain::Watch, - T::Target: 'static + BroadcasterInterface, - K::Target: 'static + KeysInterface, - F::Target: 'static + FeeEstimator, - L::Target: 'static + Logger, - Fun: Fn(&ChannelManager) -> Result<(), std::io::Error>, -{ - fn persist_manager(&self, channel_manager: &ChannelManager) -> Result<(), std::io::Error> { - self(channel_manager) - } -} - -/// Decorates an [`EventHandler`] with common functionality provided by standard [`EventHandler`]s. -struct DecoratingEventHandler< - E: EventHandler, - N: Deref>, - G: Deref, - A: Deref, - L: Deref, -> where - A::Target: chain::Access, - L::Target: Logger, -{ - event_handler: E, - net_graph_msg_handler: Option, -} - -impl< - E: EventHandler, - N: Deref>, - G: Deref, - A: Deref, - L: Deref, - > EventHandler for DecoratingEventHandler -where - A::Target: chain::Access, - L::Target: Logger, -{ - fn handle_event(&self, event: &Event) { - if let Some(event_handler) = &self.net_graph_msg_handler { - event_handler.handle_event(event); - } - self.event_handler.handle_event(event); - } -} - -impl BackgroundProcessor { - /// Start a background thread that takes care of responsibilities enumerated in the [top-level - /// documentation]. - /// - /// The thread runs indefinitely unless the object is dropped, [`stop`] is called, or - /// `persist_manager` returns an error. In case of an error, the error is retrieved by calling - /// either [`join`] or [`stop`]. - /// - /// # Data Persistence - /// - /// `persist_manager` is responsible for writing out the [`ChannelManager`] to disk, and/or - /// uploading to one or more backup services. See [`ChannelManager::write`] for writing out a - /// [`ChannelManager`]. See [`LightningPersister::persist_manager`] for Rust-Lightning's - /// provided implementation. - /// - /// Typically, users should either implement [`ChannelManagerPersister`] to never return an - /// error or call [`join`] and handle any error that may arise. For the latter case, - /// `BackgroundProcessor` must be restarted by calling `start` again after handling the error. - /// - /// # Event Handling - /// - /// `event_handler` is responsible for handling events that users should be notified of (e.g., - /// payment failed). [`BackgroundProcessor`] may decorate the given [`EventHandler`] with common - /// functionality implemented by other handlers. - /// * [`NetGraphMsgHandler`] if given will update the [`NetworkGraph`] based on payment failures. - /// - /// [top-level documentation]: BackgroundProcessor - /// [`join`]: Self::join - /// [`stop`]: Self::stop - /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager - /// [`ChannelManager::write`]: lightning::ln::channelmanager::ChannelManager#impl-Writeable - /// [`LightningPersister::persist_manager`]: lightning_persister::LightningPersister::persist_manager - /// [`NetworkGraph`]: lightning::routing::network_graph::NetworkGraph - pub fn start< - Signer: 'static + Sign, - CA: 'static + Deref + Send + Sync, - CF: 'static + Deref + Send + Sync, - CW: 'static + Deref + Send + Sync, - T: 'static + Deref + Send + Sync, - K: 'static + Deref + Send + Sync, - F: 'static + Deref + Send + Sync, - G: 'static + Deref + Send + Sync, - L: 'static + Deref + Send + Sync, - P: 'static + Deref + Send + Sync, - Descriptor: 'static + SocketDescriptor + Send + Sync, - CMH: 'static + Deref + Send + Sync, - RMH: 'static + Deref + Send + Sync, - EH: 'static + EventHandler + Send, - CMP: 'static + Send + ChannelManagerPersister, - M: 'static + Deref> + Send + Sync, - CM: 'static + Deref> + Send + Sync, - NG: 'static + Deref> + Send + Sync, - UMH: 'static + Deref + Send + Sync, - PM: 'static + Deref> + Send + Sync, - >( - persister: CMP, - event_handler: EH, - chain_monitor: M, - channel_manager: CM, - net_graph_msg_handler: Option, - peer_manager: PM, - logger: L, - ) -> Self - where - CA::Target: 'static + chain::Access, - CF::Target: 'static + chain::Filter, - CW::Target: 'static + chain::Watch, - T::Target: 'static + BroadcasterInterface, - K::Target: 'static + KeysInterface, - F::Target: 'static + FeeEstimator, - L::Target: 'static + Logger, - P::Target: 'static + Persist, - CMH::Target: 'static + ChannelMessageHandler, - RMH::Target: 'static + RoutingMessageHandler, - UMH::Target: 'static + CustomMessageHandler, - { - let stop_thread = Arc::new(AtomicBool::new(false)); - let stop_thread_clone = stop_thread.clone(); - let handle = thread::spawn(move || -> Result<(), std::io::Error> { - let event_handler = DecoratingEventHandler { - event_handler, - net_graph_msg_handler: net_graph_msg_handler.as_deref(), - }; - - log_trace!(logger, "Calling ChannelManager's timer_tick_occurred on startup"); - channel_manager.timer_tick_occurred(); - - let mut last_freshness_call = Instant::now(); - let mut last_ping_call = Instant::now(); - let mut last_prune_call = Instant::now(); - let mut have_pruned = false; - - loop { - peer_manager.process_events(); // Note that this may block on ChannelManager's locking - channel_manager.process_pending_events(&event_handler); - chain_monitor.process_pending_events(&event_handler); - - // We wait up to 100ms, but track how long it takes to detect being put to sleep, - // see `await_start`'s use below. - let await_start = Instant::now(); - let updates_available = channel_manager.await_persistable_update_timeout(Duration::from_millis(100)); - let await_time = await_start.elapsed(); - - if updates_available { - log_trace!(logger, "Persisting ChannelManager..."); - persister.persist_manager(&*channel_manager)?; - log_trace!(logger, "Done persisting ChannelManager."); - } - // Exit the loop if the background processor was requested to stop. - if stop_thread.load(Ordering::Acquire) { - log_trace!(logger, "Terminating background processor."); - break; - } - if last_freshness_call.elapsed().as_secs() > FRESHNESS_TIMER { - log_trace!(logger, "Calling ChannelManager's timer_tick_occurred"); - channel_manager.timer_tick_occurred(); - last_freshness_call = Instant::now(); - } - if await_time > Duration::from_secs(1) { - // On various platforms, we may be starved of CPU cycles for several reasons. - // E.g. on iOS, if we've been in the background, we will be entirely paused. - // Similarly, if we're on a desktop platform and the device has been asleep, we - // may not get any cycles. - // We detect this by checking if our max-100ms-sleep, above, ran longer than a - // full second, at which point we assume sockets may have been killed (they - // appear to be at least on some platforms, even if it has only been a second). - // Note that we have to take care to not get here just because user event - // processing was slow at the top of the loop. For example, the sample client - // may call Bitcoin Core RPCs during event handling, which very often takes - // more than a handful of seconds to complete, and shouldn't disconnect all our - // peers. - log_trace!(logger, "100ms sleep took more than a second, disconnecting peers."); - peer_manager.disconnect_all_peers(); - last_ping_call = Instant::now(); - } else if last_ping_call.elapsed().as_secs() > PING_TIMER { - log_trace!(logger, "Calling PeerManager's timer_tick_occurred"); - peer_manager.timer_tick_occurred(); - last_ping_call = Instant::now(); - } - - // Note that we want to run a graph prune once not long after startup before - // falling back to our usual hourly prunes. This avoids short-lived clients never - // pruning their network graph. We run once 60 seconds after startup before - // continuing our normal cadence. - if last_prune_call.elapsed().as_secs() > if have_pruned { NETWORK_PRUNE_TIMER } else { 60 } { - if let Some(ref handler) = net_graph_msg_handler { - log_trace!(logger, "Pruning network graph of stale entries"); - handler.network_graph().remove_stale_channels(); - last_prune_call = Instant::now(); - have_pruned = true; - } - } - } - // After we exit, ensure we persist the ChannelManager one final time - this avoids - // some races where users quit while channel updates were in-flight, with - // ChannelMonitor update(s) persisted without a corresponding ChannelManager update. - persister.persist_manager(&*channel_manager) - }); - Self { - stop_thread: stop_thread_clone, - thread_handle: Some(handle), - } - } - - /// Join `BackgroundProcessor`'s thread, returning any error that occurred while persisting - /// [`ChannelManager`]. - /// - /// # Panics - /// - /// This function panics if the background thread has panicked such as while persisting or - /// handling events. - /// - /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager - pub fn join(mut self) -> Result<(), std::io::Error> { - assert!(self.thread_handle.is_some()); - self.join_thread() - } - - /// Stop `BackgroundProcessor`'s thread, returning any error that occurred while persisting - /// [`ChannelManager`]. - /// - /// # Panics - /// - /// This function panics if the background thread has panicked such as while persisting or - /// handling events. - /// - /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager - pub fn stop(mut self) -> Result<(), std::io::Error> { - assert!(self.thread_handle.is_some()); - self.stop_and_join_thread() - } - - fn stop_and_join_thread(&mut self) -> Result<(), std::io::Error> { - self.stop_thread.store(true, Ordering::Release); - self.join_thread() - } - - fn join_thread(&mut self) -> Result<(), std::io::Error> { - match self.thread_handle.take() { - Some(handle) => handle.join().unwrap(), - None => Ok(()), - } - } -} - -impl Drop for BackgroundProcessor { - fn drop(&mut self) { self.stop_and_join_thread().unwrap(); } -} - -#[cfg(test)] -mod tests { - use super::{BackgroundProcessor, FRESHNESS_TIMER}; - use bitcoin::blockdata::block::BlockHeader; - use bitcoin::blockdata::constants::genesis_block; - use bitcoin::blockdata::transaction::{Transaction, TxOut}; - use bitcoin::network::constants::Network; - use db_common::sqlite::rusqlite::Connection; - use lightning::chain::channelmonitor::ANTI_REORG_DELAY; - use lightning::chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Recipient}; - use lightning::chain::transaction::OutPoint; - use lightning::chain::{chainmonitor, BestBlock, Confirm}; - use lightning::get_event_msg; - use lightning::ln::channelmanager::{ChainParameters, ChannelManager, SimpleArcChannelManager, BREAKDOWN_TIMEOUT}; - use lightning::ln::features::InitFeatures; - use lightning::ln::msgs::{ChannelMessageHandler, Init}; - use lightning::ln::peer_handler::{IgnoringMessageHandler, MessageHandler, PeerManager, SocketDescriptor}; - use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph}; - use lightning::util::config::UserConfig; - use lightning::util::events::{Event, MessageSendEvent, MessageSendEventsProvider}; - use lightning::util::ser::Writeable; - use lightning::util::test_utils; - use lightning_invoice::payment::{InvoicePayer, RetryAttempts}; - use lightning_invoice::utils::DefaultRouter; - use lightning_persister::LightningPersister; - use std::fs; - use std::path::PathBuf; - use std::sync::{Arc, Mutex}; - use std::time::Duration; - - const EVENT_DEADLINE: u64 = 5 * FRESHNESS_TIMER; - - #[derive(Clone, Eq, Hash, PartialEq)] - struct TestDescriptor {} - impl SocketDescriptor for TestDescriptor { - fn send_data(&mut self, _data: &[u8], _resume_read: bool) -> usize { 0 } - - fn disconnect_socket(&mut self) {} - } - - type ChainMonitor = chainmonitor::ChainMonitor< - InMemorySigner, - Arc, - Arc, - Arc, - Arc, - Arc, - >; - - struct Node { - node: Arc< - SimpleArcChannelManager< - ChainMonitor, - test_utils::TestBroadcaster, - test_utils::TestFeeEstimator, - test_utils::TestLogger, - >, - >, - net_graph_msg_handler: Option< - Arc, Arc, Arc>>, - >, - peer_manager: Arc< - PeerManager< - TestDescriptor, - Arc, - Arc, - Arc, - IgnoringMessageHandler, - >, - >, - chain_monitor: Arc, - persister: Arc, - tx_broadcaster: Arc, - network_graph: Arc, - logger: Arc, - best_block: BestBlock, - } - - impl Drop for Node { - fn drop(&mut self) { - let data_dir = self.persister.main_path(); - match fs::remove_dir_all(data_dir.clone()) { - Err(e) => println!( - "Failed to remove test persister directory {}: {}", - data_dir.to_str().unwrap(), - e - ), - _ => {}, - } - } - } - - fn get_full_filepath(filepath: String, filename: String) -> String { - let mut path = PathBuf::from(filepath); - path.push(filename); - path.to_str().unwrap().to_string() - } - - fn create_nodes(num_nodes: usize, persist_dir: String) -> Vec { - let mut nodes = Vec::new(); - for i in 0..num_nodes { - let tx_broadcaster = Arc::new(test_utils::TestBroadcaster { - txn_broadcasted: Mutex::new(Vec::new()), - blocks: Arc::new(Mutex::new(Vec::new())), - }); - let fee_estimator = Arc::new(test_utils::TestFeeEstimator { - sat_per_kw: Mutex::new(253), - }); - let chain_source = Arc::new(test_utils::TestChainSource::new(Network::Testnet)); - let logger = Arc::new(test_utils::TestLogger::with_id(format!("node {}", i))); - let persister = Arc::new(LightningPersister::new( - format!("node_{}_ticker", i), - PathBuf::from(format!("{}_persister_{}", persist_dir, i)), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - )); - let seed = [i as u8; 32]; - let network = Network::Testnet; - let genesis_block = genesis_block(network); - let now = Duration::from_secs(genesis_block.header.time as u64); - let keys_manager = Arc::new(KeysManager::new(&seed, now.as_secs(), now.subsec_nanos())); - let chain_monitor = Arc::new(chainmonitor::ChainMonitor::new( - Some(chain_source.clone()), - tx_broadcaster.clone(), - logger.clone(), - fee_estimator.clone(), - persister.clone(), - )); - let best_block = BestBlock::from_genesis(network); - let params = ChainParameters { network, best_block }; - let manager = Arc::new(ChannelManager::new( - fee_estimator.clone(), - chain_monitor.clone(), - tx_broadcaster.clone(), - logger.clone(), - keys_manager.clone(), - UserConfig::default(), - params, - )); - let network_graph = Arc::new(NetworkGraph::new(genesis_block.header.block_hash())); - let net_graph_msg_handler = Some(Arc::new(NetGraphMsgHandler::new( - network_graph.clone(), - Some(chain_source.clone()), - logger.clone(), - ))); - let msg_handler = MessageHandler { - chan_handler: Arc::new(test_utils::TestChannelMessageHandler::new()), - route_handler: Arc::new(test_utils::TestRoutingMessageHandler::new()), - }; - let peer_manager = Arc::new(PeerManager::new( - msg_handler, - keys_manager.get_node_secret(Recipient::Node).unwrap(), - &seed, - logger.clone(), - IgnoringMessageHandler {}, - )); - let node = Node { - node: manager, - net_graph_msg_handler, - peer_manager, - chain_monitor, - persister, - tx_broadcaster, - network_graph, - logger, - best_block, - }; - nodes.push(node); - } - - for i in 0..num_nodes { - for j in (i + 1)..num_nodes { - nodes[i].node.peer_connected(&nodes[j].node.get_our_node_id(), &Init { - features: InitFeatures::known(), - remote_network_address: None, - }); - nodes[j].node.peer_connected(&nodes[i].node.get_our_node_id(), &Init { - features: InitFeatures::known(), - remote_network_address: None, - }); - } - } - - nodes - } - - macro_rules! open_channel { - ($node_a: expr, $node_b: expr, $channel_value: expr) => {{ - begin_open_channel!($node_a, $node_b, $channel_value); - let events = $node_a.node.get_and_clear_pending_events(); - assert_eq!(events.len(), 1); - let (temporary_channel_id, tx) = handle_funding_generation_ready!(&events[0], $channel_value); - end_open_channel!($node_a, $node_b, temporary_channel_id, tx); - tx - }}; - } - - macro_rules! begin_open_channel { - ($node_a: expr, $node_b: expr, $channel_value: expr) => {{ - $node_a - .node - .create_channel($node_b.node.get_our_node_id(), $channel_value, 100, 42, None) - .unwrap(); - $node_b.node.handle_open_channel( - &$node_a.node.get_our_node_id(), - InitFeatures::known(), - &get_event_msg!( - $node_a, - MessageSendEvent::SendOpenChannel, - $node_b.node.get_our_node_id() - ), - ); - $node_a.node.handle_accept_channel( - &$node_b.node.get_our_node_id(), - InitFeatures::known(), - &get_event_msg!( - $node_b, - MessageSendEvent::SendAcceptChannel, - $node_a.node.get_our_node_id() - ), - ); - }}; - } - - macro_rules! handle_funding_generation_ready { - ($event: expr, $channel_value: expr) => {{ - match $event { - &Event::FundingGenerationReady { - temporary_channel_id, - channel_value_satoshis, - ref output_script, - user_channel_id, - } => { - assert_eq!(channel_value_satoshis, $channel_value); - assert_eq!(user_channel_id, 42); - - let tx = Transaction { - version: 1 as i32, - lock_time: 0, - input: Vec::new(), - output: vec![TxOut { - value: channel_value_satoshis, - script_pubkey: output_script.clone(), - }], - }; - (temporary_channel_id, tx) - }, - _ => panic!("Unexpected event"), - } - }}; - } - - macro_rules! end_open_channel { - ($node_a: expr, $node_b: expr, $temporary_channel_id: expr, $tx: expr) => {{ - $node_a - .node - .funding_transaction_generated(&$temporary_channel_id, $tx.clone()) - .unwrap(); - $node_b.node.handle_funding_created( - &$node_a.node.get_our_node_id(), - &get_event_msg!( - $node_a, - MessageSendEvent::SendFundingCreated, - $node_b.node.get_our_node_id() - ), - ); - $node_a.node.handle_funding_signed( - &$node_b.node.get_our_node_id(), - &get_event_msg!( - $node_b, - MessageSendEvent::SendFundingSigned, - $node_a.node.get_our_node_id() - ), - ); - }}; - } - - fn confirm_transaction_depth(node: &mut Node, tx: &Transaction, depth: u32) { - for i in 1..=depth { - let prev_blockhash = node.best_block.block_hash(); - let height = node.best_block.height() + 1; - let header = BlockHeader { - version: 0x20000000, - prev_blockhash, - merkle_root: Default::default(), - time: height, - bits: 42, - nonce: 42, - }; - let txdata = vec![(0, tx)]; - node.best_block = BestBlock::new(header.block_hash(), height); - match i { - 1 => { - node.node.transactions_confirmed(&header, &txdata, height); - node.chain_monitor.transactions_confirmed(&header, &txdata, height); - }, - x if x == depth => { - node.node.best_block_updated(&header, height); - node.chain_monitor.best_block_updated(&header, height); - }, - _ => {}, - } - } - } - fn confirm_transaction(node: &mut Node, tx: &Transaction) { confirm_transaction_depth(node, tx, ANTI_REORG_DELAY); } - - #[test] - fn test_background_processor() { - // Test that when a new channel is created, the ChannelManager needs to be re-persisted with - // updates. Also test that when new updates are available, the manager signals that it needs - // re-persistence and is successfully re-persisted. - let nodes = create_nodes(2, "test_background_processor".to_string()); - - // Go through the channel creation process so that each node has something to persist. Since - // open_channel consumes events, it must complete before starting BackgroundProcessor to - // avoid a race with processing events. - let tx = open_channel!(nodes[0], nodes[1], 100000); - - // Initiate the background processors to watch each node. - let node_0_persister = nodes[0].persister.clone(); - let persister = move |node: &ChannelManager< - InMemorySigner, - Arc, - Arc, - Arc, - Arc, - Arc, - >| node_0_persister.persist_manager(node); - let event_handler = |_: &_| {}; - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - - macro_rules! check_persisted_data { - ($node: expr, $filepath: expr, $expected_bytes: expr) => { - loop { - $expected_bytes.clear(); - match $node.write(&mut $expected_bytes) { - Ok(()) => match std::fs::read($filepath) { - Ok(bytes) => { - if bytes == $expected_bytes { - break; - } else { - continue; - } - }, - Err(_) => continue, - }, - Err(e) => panic!("Unexpected error: {}", e), - } - } - }; - } - - // Check that the initial channel manager data is persisted as expected. - let filepath = get_full_filepath( - "test_background_processor_persister_0".to_string(), - "manager".to_string(), - ); - let mut expected_bytes = Vec::new(); - check_persisted_data!(nodes[0].node, filepath.clone(), expected_bytes); - loop { - if !nodes[0].node.get_persistence_condvar_value() { - break; - } - } - - // Force-close the channel. - nodes[0] - .node - .force_close_channel( - &OutPoint { - txid: tx.txid(), - index: 0, - } - .to_channel_id(), - ) - .unwrap(); - - // Check that the force-close updates are persisted. - let mut expected_bytes = Vec::new(); - check_persisted_data!(nodes[0].node, filepath.clone(), expected_bytes); - loop { - if !nodes[0].node.get_persistence_condvar_value() { - break; - } - } - - assert!(bg_processor.stop().is_ok()); - } - - #[test] - fn test_timer_tick_called() { - // Test that ChannelManager's and PeerManager's `timer_tick_occurred` is called every - // `FRESHNESS_TIMER`. - let nodes = create_nodes(1, "test_timer_tick_called".to_string()); - let node_0_persister = nodes[0].persister.clone(); - let persister = move |node: &ChannelManager< - InMemorySigner, - Arc, - Arc, - Arc, - Arc, - Arc, - >| node_0_persister.persist_manager(node); - let event_handler = |_: &_| {}; - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - loop { - let log_entries = nodes[0].logger.lines.lock().unwrap(); - let desired_log = "Calling ChannelManager's timer_tick_occurred".to_string(); - let second_desired_log = "Calling PeerManager's timer_tick_occurred".to_string(); - if log_entries - .get(&("lightning_background_processor".to_string(), desired_log)) - .is_some() - && log_entries - .get(&("lightning_background_processor".to_string(), second_desired_log)) - .is_some() - { - break; - } - } - - assert!(bg_processor.stop().is_ok()); - } - - #[test] - fn test_persist_error() { - // Test that if we encounter an error during manager persistence, the thread panics. - let nodes = create_nodes(2, "test_persist_error".to_string()); - open_channel!(nodes[0], nodes[1], 100000); - - let persister = |_: &_| Err(std::io::Error::new(std::io::ErrorKind::Other, "test")); - let event_handler = |_: &_| {}; - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - match bg_processor.join() { - Ok(_) => panic!("Expected error persisting manager"), - Err(e) => { - assert_eq!(e.kind(), std::io::ErrorKind::Other); - assert_eq!(e.get_ref().unwrap().to_string(), "test"); - }, - } - } - - #[test] - fn test_background_event_handling() { - let mut nodes = create_nodes(2, "test_background_event_handling".to_string()); - let channel_value = 100000; - let node_0_persister = nodes[0].persister.clone(); - let persister = move |node: &_| node_0_persister.persist_manager(node); - - // Set up a background event handler for FundingGenerationReady events. - let (sender, receiver) = std::sync::mpsc::sync_channel(1); - let event_handler = move |event: &Event| { - sender - .send(handle_funding_generation_ready!(event, channel_value)) - .unwrap(); - }; - let bg_processor = BackgroundProcessor::start( - persister.clone(), - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - - // Open a channel and check that the FundingGenerationReady event was handled. - begin_open_channel!(nodes[0], nodes[1], channel_value); - let (temporary_channel_id, funding_tx) = receiver - .recv_timeout(Duration::from_secs(EVENT_DEADLINE)) - .expect("FundingGenerationReady not handled within deadline"); - end_open_channel!(nodes[0], nodes[1], temporary_channel_id, funding_tx); - - // Confirm the funding transaction. - confirm_transaction(&mut nodes[0], &funding_tx); - let as_funding = get_event_msg!( - nodes[0], - MessageSendEvent::SendFundingLocked, - nodes[1].node.get_our_node_id() - ); - confirm_transaction(&mut nodes[1], &funding_tx); - let bs_funding = get_event_msg!( - nodes[1], - MessageSendEvent::SendFundingLocked, - nodes[0].node.get_our_node_id() - ); - nodes[0] - .node - .handle_funding_locked(&nodes[1].node.get_our_node_id(), &bs_funding); - let _as_channel_update = get_event_msg!( - nodes[0], - MessageSendEvent::SendChannelUpdate, - nodes[1].node.get_our_node_id() - ); - nodes[1] - .node - .handle_funding_locked(&nodes[0].node.get_our_node_id(), &as_funding); - let _bs_channel_update = get_event_msg!( - nodes[1], - MessageSendEvent::SendChannelUpdate, - nodes[0].node.get_our_node_id() - ); - - assert!(bg_processor.stop().is_ok()); - - // Set up a background event handler for SpendableOutputs events. - let (sender, receiver) = std::sync::mpsc::sync_channel(1); - let event_handler = move |event: &Event| sender.send(event.clone()).unwrap(); - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - - // Force close the channel and check that the SpendableOutputs event was handled. - nodes[0] - .node - .force_close_channel(&nodes[0].node.list_channels()[0].channel_id) - .unwrap(); - let commitment_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().pop().unwrap(); - confirm_transaction_depth(&mut nodes[0], &commitment_tx, BREAKDOWN_TIMEOUT as u32); - let event = receiver - .recv_timeout(Duration::from_secs(EVENT_DEADLINE)) - .expect("SpendableOutputs not handled within deadline"); - match event { - Event::SpendableOutputs { .. } => {}, - Event::ChannelClosed { .. } => {}, - _ => panic!("Unexpected event: {:?}", event), - } - - assert!(bg_processor.stop().is_ok()); - } - - #[test] - fn test_invoice_payer() { - let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet); - let random_seed_bytes = keys_manager.get_secure_random_bytes(); - let nodes = create_nodes(2, "test_invoice_payer".to_string()); - - // Initiate the background processors to watch each node. - let node_0_persister = nodes[0].persister.clone(); - let persister = move |node: &ChannelManager< - InMemorySigner, - Arc, - Arc, - Arc, - Arc, - Arc, - >| node_0_persister.persist_manager(node); - let router = DefaultRouter::new( - Arc::clone(&nodes[0].network_graph), - Arc::clone(&nodes[0].logger), - random_seed_bytes, - ); - let scorer = Arc::new(Mutex::new(test_utils::TestScorer::with_penalty(0))); - let invoice_payer = Arc::new(InvoicePayer::new( - Arc::clone(&nodes[0].node), - router, - scorer, - Arc::clone(&nodes[0].logger), - |_: &_| {}, - RetryAttempts(2), - )); - let event_handler = Arc::clone(&invoice_payer); - let bg_processor = BackgroundProcessor::start( - persister, - event_handler, - nodes[0].chain_monitor.clone(), - nodes[0].node.clone(), - nodes[0].net_graph_msg_handler.clone(), - nodes[0].peer_manager.clone(), - nodes[0].logger.clone(), - ); - assert!(bg_processor.stop().is_ok()); - } -} diff --git a/mm2src/coins/lightning_persister/Cargo.toml b/mm2src/coins/lightning_persister/Cargo.toml deleted file mode 100644 index 32b5d7eb1d..0000000000 --- a/mm2src/coins/lightning_persister/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "lightning-persister" -version = "0.0.106" -edition = "2018" -authors = ["Valentine Wallace", "Matt Corallo"] -license = "MIT OR Apache-2.0" -repository = "https://github.com/lightningdevkit/rust-lightning/" -description = """ -Utilities to manage Rust-Lightning channel data persistence and retrieval. -""" - -[dependencies] -async-trait = "0.1" -bitcoin = "0.27.1" -common = { path = "../../common" } -mm2_io = { path = "../../mm2_io" } -db_common = { path = "../../db_common" } -derive_more = "0.99" -hex = "0.4.2" -lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106" } -libc = "0.2" -parking_lot = { version = "0.12.0", features = ["nightly"] } -secp256k1 = { version = "0.20" } -serde = "1.0" -serde_json = "1.0" - -[target.'cfg(windows)'.dependencies] -winapi = { version = "0.3", features = ["winbase"] } - -[dev-dependencies] -lightning = { git = "https://github.com/shamardy/rust-lightning", branch = "0.0.106", features = ["_test_utils"] } -rand = { version = "0.7", features = ["std", "small_rng"] } \ No newline at end of file diff --git a/mm2src/coins/lightning_persister/src/lib.rs b/mm2src/coins/lightning_persister/src/lib.rs deleted file mode 100644 index 303205c26f..0000000000 --- a/mm2src/coins/lightning_persister/src/lib.rs +++ /dev/null @@ -1,2097 +0,0 @@ -//! Utilities that handle persisting Rust-Lightning data to disk via standard filesystem APIs. - -#![feature(io_error_more)] - -pub mod storage; -mod util; - -extern crate async_trait; -extern crate bitcoin; -extern crate common; -extern crate libc; -extern crate lightning; -extern crate secp256k1; -extern crate serde_json; - -use crate::storage::{ChannelType, ChannelVisibility, ClosedChannelsFilter, DbStorage, FileSystemStorage, - GetClosedChannelsResult, GetPaymentsResult, HTLCStatus, NodesAddressesMap, - NodesAddressesMapShared, PaymentInfo, PaymentType, PaymentsFilter, Scorer, SqlChannelDetails}; -use crate::util::DiskWriteable; -use async_trait::async_trait; -use bitcoin::blockdata::constants::genesis_block; -use bitcoin::hash_types::{BlockHash, Txid}; -use bitcoin::hashes::hex::{FromHex, ToHex}; -use bitcoin::Network; -use common::{async_blocking, PagingOptionsEnum}; -use db_common::sqlite::rusqlite::{Error as SqlError, Row, ToSql, NO_PARAMS}; -use db_common::sqlite::sql_builder::SqlBuilder; -use db_common::sqlite::{h256_option_slice_from_row, h256_slice_from_row, offset_by_id, query_single_row, - sql_text_conversion_err, string_from_row, validate_table_name, SqliteConnShared, - CHECK_TABLE_EXISTS_SQL}; -use lightning::chain; -use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; -use lightning::chain::chainmonitor; -use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; -use lightning::chain::keysinterface::{KeysInterface, Sign}; -use lightning::chain::transaction::OutPoint; -use lightning::ln::channelmanager::ChannelManager; -use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; -use lightning::routing::network_graph::NetworkGraph; -use lightning::routing::scoring::ProbabilisticScoringParameters; -use lightning::util::logger::Logger; -use lightning::util::ser::{Readable, ReadableArgs, Writeable}; -use mm2_io::fs::check_dir_operations; -use secp256k1::PublicKey; -use std::collections::HashMap; -use std::convert::TryInto; -use std::fs; -use std::io::{BufReader, BufWriter, Cursor, Error}; -use std::net::SocketAddr; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::str::FromStr; -use std::sync::{Arc, Mutex}; - -/// LightningPersister persists channel data on disk, where each channel's -/// data is stored in a file named after its funding outpoint. -/// It is also used to persist payments and channels history to sqlite database. -/// -/// Warning: this module does the best it can with calls to persist data, but it -/// can only guarantee that the data is passed to the drive. It is up to the -/// drive manufacturers to do the actual persistence properly, which they often -/// don't (especially on consumer-grade hardware). Therefore, it is up to the -/// user to validate their entire storage stack, to ensure the writes are -/// persistent. -/// Corollary: especially when dealing with larger amounts of money, it is best -/// practice to have multiple channel data backups and not rely only on one -/// LightningPersister. - -pub struct LightningPersister { - storage_ticker: String, - main_path: PathBuf, - backup_path: Option, - sqlite_connection: SqliteConnShared, -} - -impl DiskWriteable for ChannelMonitor { - fn write_to_file(&self, writer: &mut fs::File) -> Result<(), Error> { self.write(writer) } -} - -impl DiskWriteable - for ChannelManager -where - M::Target: chain::Watch, - T::Target: BroadcasterInterface, - K::Target: KeysInterface, - F::Target: FeeEstimator, - L::Target: Logger, -{ - fn write_to_file(&self, writer: &mut fs::File) -> Result<(), std::io::Error> { self.write(writer) } -} - -fn channels_history_table(ticker: &str) -> String { ticker.to_owned() + "_channels_history" } - -fn payments_history_table(ticker: &str) -> String { ticker.to_owned() + "_payments_history" } - -fn create_channels_history_table_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "CREATE TABLE IF NOT EXISTS {} ( - id INTEGER NOT NULL PRIMARY KEY, - rpc_id INTEGER NOT NULL UNIQUE, - channel_id VARCHAR(255) NOT NULL, - counterparty_node_id VARCHAR(255) NOT NULL, - funding_tx VARCHAR(255), - funding_value INTEGER, - funding_generated_in_block Integer, - closing_tx VARCHAR(255), - closure_reason TEXT, - claiming_tx VARCHAR(255), - claimed_balance REAL, - is_outbound INTEGER NOT NULL, - is_public INTEGER NOT NULL, - is_closed INTEGER NOT NULL, - created_at INTEGER NOT NULL, - closed_at INTEGER - );", - table_name - ); - - Ok(sql) -} - -fn create_payments_history_table_sql(for_coin: &str) -> Result { - let table_name = payments_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "CREATE TABLE IF NOT EXISTS {} ( - id INTEGER NOT NULL PRIMARY KEY, - payment_hash VARCHAR(255) NOT NULL UNIQUE, - destination VARCHAR(255), - description VARCHAR(641) NOT NULL, - preimage VARCHAR(255), - secret VARCHAR(255), - amount_msat INTEGER, - fee_paid_msat INTEGER, - is_outbound INTEGER NOT NULL, - status VARCHAR(255) NOT NULL, - created_at INTEGER NOT NULL, - last_updated INTEGER NOT NULL - );", - table_name - ); - - Ok(sql) -} - -fn insert_channel_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "INSERT INTO {} ( - rpc_id, - channel_id, - counterparty_node_id, - is_outbound, - is_public, - is_closed, - created_at - ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7 - );", - table_name - ); - - Ok(sql) -} - -fn upsert_payment_sql(for_coin: &str) -> Result { - let table_name = payments_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "INSERT OR REPLACE INTO {} ( - payment_hash, - destination, - description, - preimage, - secret, - amount_msat, - fee_paid_msat, - is_outbound, - status, - created_at, - last_updated - ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 - );", - table_name - ); - - Ok(sql) -} - -fn select_channel_by_rpc_id_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "SELECT - rpc_id, - channel_id, - counterparty_node_id, - funding_tx, - funding_value, - funding_generated_in_block, - closing_tx, - closure_reason, - claiming_tx, - claimed_balance, - is_outbound, - is_public, - is_closed, - created_at, - closed_at - FROM - {} - WHERE - rpc_id=?1", - table_name - ); - - Ok(sql) -} - -fn select_payment_by_hash_sql(for_coin: &str) -> Result { - let table_name = payments_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "SELECT - payment_hash, - destination, - description, - preimage, - secret, - amount_msat, - fee_paid_msat, - status, - is_outbound, - created_at, - last_updated - FROM - {} - WHERE - payment_hash=?1;", - table_name - ); - - Ok(sql) -} - -fn channel_details_from_row(row: &Row<'_>) -> Result { - let channel_details = SqlChannelDetails { - rpc_id: row.get::<_, u32>(0)? as u64, - channel_id: row.get(1)?, - counterparty_node_id: row.get(2)?, - funding_tx: row.get(3)?, - funding_value: row.get::<_, Option>(4)?.map(|v| v as u64), - funding_generated_in_block: row.get::<_, Option>(5)?.map(|v| v as u64), - closing_tx: row.get(6)?, - closure_reason: row.get(7)?, - claiming_tx: row.get(8)?, - claimed_balance: row.get::<_, Option>(9)?, - is_outbound: row.get(10)?, - is_public: row.get(11)?, - is_closed: row.get(12)?, - created_at: row.get::<_, u32>(13)? as u64, - closed_at: row.get::<_, Option>(14)?.map(|t| t as u64), - }; - Ok(channel_details) -} - -fn payment_info_from_row(row: &Row<'_>) -> Result { - let is_outbound = row.get::<_, bool>(8)?; - let payment_type = if is_outbound { - PaymentType::OutboundPayment { - destination: PublicKey::from_str(&row.get::<_, String>(1)?).map_err(|e| sql_text_conversion_err(1, e))?, - } - } else { - PaymentType::InboundPayment - }; - - let payment_info = PaymentInfo { - payment_hash: PaymentHash(h256_slice_from_row::(row, 0)?), - payment_type, - description: row.get(2)?, - preimage: h256_option_slice_from_row::(row, 3)?.map(PaymentPreimage), - secret: h256_option_slice_from_row::(row, 4)?.map(PaymentSecret), - amt_msat: row.get::<_, Option>(5)?.map(|v| v as u64), - fee_paid_msat: row.get::<_, Option>(6)?.map(|v| v as u64), - status: HTLCStatus::from_str(&row.get::<_, String>(7)?)?, - created_at: row.get::<_, u32>(9)? as u64, - last_updated: row.get::<_, u32>(10)? as u64, - }; - Ok(payment_info) -} - -fn get_last_channel_rpc_id_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!("SELECT IFNULL(MAX(rpc_id), 0) FROM {};", table_name); - - Ok(sql) -} - -fn update_funding_tx_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "UPDATE {} SET - funding_tx = ?1, - funding_value = ?2, - funding_generated_in_block = ?3 - WHERE - rpc_id = ?4;", - table_name - ); - - Ok(sql) -} - -fn update_funding_tx_block_height_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "UPDATE {} SET funding_generated_in_block = ?1 WHERE funding_tx = ?2;", - table_name - ); - - Ok(sql) -} - -fn update_channel_to_closed_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "UPDATE {} SET closure_reason = ?1, is_closed = ?2, closed_at = ?3 WHERE rpc_id = ?4;", - table_name - ); - - Ok(sql) -} - -fn update_closing_tx_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!("UPDATE {} SET closing_tx = ?1 WHERE rpc_id = ?2;", table_name); - - Ok(sql) -} - -fn get_channels_builder_preimage(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let mut sql_builder = SqlBuilder::select_from(table_name); - sql_builder.and_where("is_closed = 1"); - Ok(sql_builder) -} - -fn add_fields_to_get_channels_sql_builder(sql_builder: &mut SqlBuilder) { - sql_builder - .field("rpc_id") - .field("channel_id") - .field("counterparty_node_id") - .field("funding_tx") - .field("funding_value") - .field("funding_generated_in_block") - .field("closing_tx") - .field("closure_reason") - .field("claiming_tx") - .field("claimed_balance") - .field("is_outbound") - .field("is_public") - .field("is_closed") - .field("created_at") - .field("closed_at"); -} - -fn finalize_get_channels_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { - sql_builder.offset(offset); - sql_builder.limit(limit); - sql_builder.order_desc("closed_at"); -} - -fn apply_get_channels_filter(builder: &mut SqlBuilder, params: &mut Vec<(&str, String)>, filter: ClosedChannelsFilter) { - if let Some(channel_id) = filter.channel_id { - builder.and_where("channel_id = :channel_id"); - params.push((":channel_id", channel_id)); - } - - if let Some(counterparty_node_id) = filter.counterparty_node_id { - builder.and_where("counterparty_node_id = :counterparty_node_id"); - params.push((":counterparty_node_id", counterparty_node_id)); - } - - if let Some(funding_tx) = filter.funding_tx { - builder.and_where("funding_tx = :funding_tx"); - params.push((":funding_tx", funding_tx)); - } - - if let Some(from_funding_value) = filter.from_funding_value { - builder.and_where("funding_value >= :from_funding_value"); - params.push((":from_funding_value", from_funding_value.to_string())); - } - - if let Some(to_funding_value) = filter.to_funding_value { - builder.and_where("funding_value <= :to_funding_value"); - params.push((":to_funding_value", to_funding_value.to_string())); - } - - if let Some(closing_tx) = filter.closing_tx { - builder.and_where("closing_tx = :closing_tx"); - params.push((":closing_tx", closing_tx)); - } - - if let Some(closure_reason) = filter.closure_reason { - builder.and_where(format!("closure_reason LIKE '%{}%'", closure_reason)); - } - - if let Some(claiming_tx) = filter.claiming_tx { - builder.and_where("claiming_tx = :claiming_tx"); - params.push((":claiming_tx", claiming_tx)); - } - - if let Some(from_claimed_balance) = filter.from_claimed_balance { - builder.and_where("claimed_balance >= :from_claimed_balance"); - params.push((":from_claimed_balance", from_claimed_balance.to_string())); - } - - if let Some(to_claimed_balance) = filter.to_claimed_balance { - builder.and_where("claimed_balance <= :to_claimed_balance"); - params.push((":to_claimed_balance", to_claimed_balance.to_string())); - } - - if let Some(channel_type) = filter.channel_type { - let is_outbound = match channel_type { - ChannelType::Outbound => true as i32, - ChannelType::Inbound => false as i32, - }; - - builder.and_where("is_outbound = :is_outbound"); - params.push((":is_outbound", is_outbound.to_string())); - } - - if let Some(channel_visibility) = filter.channel_visibility { - let is_public = match channel_visibility { - ChannelVisibility::Public => true as i32, - ChannelVisibility::Private => false as i32, - }; - - builder.and_where("is_public = :is_public"); - params.push((":is_public", is_public.to_string())); - } -} - -fn get_payments_builder_preimage(for_coin: &str) -> Result { - let table_name = payments_history_table(for_coin); - validate_table_name(&table_name)?; - - Ok(SqlBuilder::select_from(table_name)) -} - -fn finalize_get_payments_sql_builder(sql_builder: &mut SqlBuilder, offset: usize, limit: usize) { - sql_builder - .field("payment_hash") - .field("destination") - .field("description") - .field("preimage") - .field("secret") - .field("amount_msat") - .field("fee_paid_msat") - .field("status") - .field("is_outbound") - .field("created_at") - .field("last_updated"); - sql_builder.offset(offset); - sql_builder.limit(limit); - sql_builder.order_desc("last_updated"); -} - -fn apply_get_payments_filter(builder: &mut SqlBuilder, params: &mut Vec<(&str, String)>, filter: PaymentsFilter) { - if let Some(payment_type) = filter.payment_type { - let (is_outbound, destination) = match payment_type { - PaymentType::OutboundPayment { destination } => (true as i32, Some(destination.to_string())), - PaymentType::InboundPayment => (false as i32, None), - }; - if let Some(dest) = destination { - builder.and_where("destination = :dest"); - params.push((":dest", dest)); - } - - builder.and_where("is_outbound = :is_outbound"); - params.push((":is_outbound", is_outbound.to_string())); - } - - if let Some(description) = filter.description { - builder.and_where(format!("description LIKE '%{}%'", description)); - } - - if let Some(status) = filter.status { - builder.and_where("status = :status"); - params.push((":status", status.to_string())); - } - - if let Some(from_amount) = filter.from_amount_msat { - builder.and_where("amount_msat >= :from_amount"); - params.push((":from_amount", from_amount.to_string())); - } - - if let Some(to_amount) = filter.to_amount_msat { - builder.and_where("amount_msat <= :to_amount"); - params.push((":to_amount", to_amount.to_string())); - } - - if let Some(from_fee) = filter.from_fee_paid_msat { - builder.and_where("fee_paid_msat >= :from_fee"); - params.push((":from_fee", from_fee.to_string())); - } - - if let Some(to_fee) = filter.to_fee_paid_msat { - builder.and_where("fee_paid_msat <= :to_fee"); - params.push((":to_fee", to_fee.to_string())); - } - - if let Some(from_time) = filter.from_timestamp { - builder.and_where("created_at >= :from_time"); - params.push((":from_time", from_time.to_string())); - } - - if let Some(to_time) = filter.to_timestamp { - builder.and_where("created_at <= :to_time"); - params.push((":to_time", to_time.to_string())); - } -} - -fn update_claiming_tx_sql(for_coin: &str) -> Result { - let table_name = channels_history_table(for_coin); - validate_table_name(&table_name)?; - - let sql = format!( - "UPDATE {} SET claiming_tx = ?1, claimed_balance = ?2 WHERE closing_tx = ?3;", - table_name - ); - - Ok(sql) -} - -impl LightningPersister { - /// Initialize a new LightningPersister and set the path to the individual channels' - /// files. - pub fn new( - storage_ticker: String, - main_path: PathBuf, - backup_path: Option, - sqlite_connection: SqliteConnShared, - ) -> Self { - Self { - storage_ticker, - main_path, - backup_path, - sqlite_connection, - } - } - - /// Get the directory which was provided when this persister was initialized. - pub fn main_path(&self) -> PathBuf { self.main_path.clone() } - - /// Get the backup directory which was provided when this persister was initialized. - pub fn backup_path(&self) -> Option { self.backup_path.clone() } - - pub(crate) fn monitor_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("monitors"); - path - } - - pub(crate) fn monitor_backup_path(&self) -> Option { - if let Some(mut backup_path) = self.backup_path() { - backup_path.push("monitors"); - return Some(backup_path); - } - None - } - - pub(crate) fn nodes_addresses_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("channel_nodes_data"); - path - } - - pub(crate) fn nodes_addresses_backup_path(&self) -> Option { - if let Some(mut backup_path) = self.backup_path() { - backup_path.push("channel_nodes_data"); - return Some(backup_path); - } - None - } - - pub(crate) fn network_graph_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("network_graph"); - path - } - - pub(crate) fn scorer_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("scorer"); - path - } - - pub fn manager_path(&self) -> PathBuf { - let mut path = self.main_path(); - path.push("manager"); - path - } - - /// Writes the provided `ChannelManager` to the path provided at `LightningPersister` - /// initialization, within a file called "manager". - pub fn persist_manager( - &self, - manager: &ChannelManager, - ) -> Result<(), std::io::Error> - where - M::Target: chain::Watch, - T::Target: BroadcasterInterface, - K::Target: KeysInterface, - F::Target: FeeEstimator, - L::Target: Logger, - { - let path = self.main_path(); - util::write_to_file(path, "manager".to_string(), manager)?; - if let Some(backup_path) = self.backup_path() { - util::write_to_file(backup_path, "manager".to_string(), manager)?; - } - Ok(()) - } - - /// Read `ChannelMonitor`s from disk. - pub fn read_channelmonitors( - &self, - keys_manager: K, - ) -> Result)>, std::io::Error> - where - K::Target: KeysInterface + Sized, - { - let path = self.monitor_path(); - if !Path::new(&path).exists() { - return Ok(Vec::new()); - } - let mut res = Vec::new(); - for file_option in fs::read_dir(path).unwrap() { - let file = file_option.unwrap(); - let owned_file_name = file.file_name(); - let filename = owned_file_name.to_str(); - if filename.is_none() || !filename.unwrap().is_ascii() || filename.unwrap().len() < 65 { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Invalid ChannelMonitor file name", - )); - } - if filename.unwrap().ends_with(".tmp") { - // If we were in the middle of committing an new update and crashed, it should be - // safe to ignore the update - we should never have returned to the caller and - // irrevocably committed to the new state in any way. - continue; - } - - let txid = Txid::from_hex(filename.unwrap().split_at(64).0); - if txid.is_err() { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Invalid tx ID in filename", - )); - } - - let index = filename.unwrap().split_at(65).1.parse::(); - if index.is_err() { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Invalid tx index in filename", - )); - } - - let contents = fs::read(&file.path())?; - let mut buffer = Cursor::new(&contents); - match <(BlockHash, ChannelMonitor)>::read(&mut buffer, &*keys_manager) { - Ok((blockhash, channel_monitor)) => { - if channel_monitor.get_funding_txo().0.txid != txid.unwrap() - || channel_monitor.get_funding_txo().0.index != index.unwrap() - { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "ChannelMonitor was stored in the wrong file", - )); - } - res.push((blockhash, channel_monitor)); - }, - Err(e) => { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Failed to deserialize ChannelMonitor: {}", e), - )) - }, - } - } - Ok(res) - } -} - -impl chainmonitor::Persist for LightningPersister { - // TODO: We really need a way for the persister to inform the user that its time to crash/shut - // down once these start returning failure. - // A PermanentFailure implies we need to shut down since we're force-closing channels without - // even broadcasting! - - fn persist_new_channel( - &self, - funding_txo: OutPoint, - monitor: &ChannelMonitor, - _update_id: chainmonitor::MonitorUpdateId, - ) -> Result<(), chain::ChannelMonitorUpdateErr> { - let filename = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); - util::write_to_file(self.monitor_path(), filename.clone(), monitor) - .map_err(|_| chain::ChannelMonitorUpdateErr::PermanentFailure)?; - if let Some(backup_path) = self.monitor_backup_path() { - util::write_to_file(backup_path, filename, monitor) - .map_err(|_| chain::ChannelMonitorUpdateErr::PermanentFailure)?; - } - Ok(()) - } - - fn update_persisted_channel( - &self, - funding_txo: OutPoint, - _update: &Option, - monitor: &ChannelMonitor, - _update_id: chainmonitor::MonitorUpdateId, - ) -> Result<(), chain::ChannelMonitorUpdateErr> { - let filename = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); - util::write_to_file(self.monitor_path(), filename.clone(), monitor) - .map_err(|_| chain::ChannelMonitorUpdateErr::PermanentFailure)?; - if let Some(backup_path) = self.monitor_backup_path() { - util::write_to_file(backup_path, filename, monitor) - .map_err(|_| chain::ChannelMonitorUpdateErr::PermanentFailure)?; - } - Ok(()) - } -} - -#[async_trait] -impl FileSystemStorage for LightningPersister { - type Error = std::io::Error; - - async fn init_fs(&self) -> Result<(), Self::Error> { - let path = self.main_path(); - let backup_path = self.backup_path(); - async_blocking(move || { - fs::create_dir_all(path.clone())?; - if let Some(path) = backup_path { - fs::create_dir_all(path.clone())?; - check_dir_operations(&path)?; - } - check_dir_operations(&path) - }) - .await - } - - async fn is_fs_initialized(&self) -> Result { - let dir_path = self.main_path(); - let backup_dir_path = self.backup_path(); - async_blocking(move || { - if !dir_path.exists() || backup_dir_path.as_ref().map(|path| !path.exists()).unwrap_or(false) { - Ok(false) - } else if !dir_path.is_dir() { - Err(std::io::Error::new( - std::io::ErrorKind::NotADirectory, - format!("{} is not a directory", dir_path.display()), - )) - } else if backup_dir_path.as_ref().map(|path| !path.is_dir()).unwrap_or(false) { - Err(std::io::Error::new( - std::io::ErrorKind::NotADirectory, - "Backup path is not a directory", - )) - } else { - let check_backup_ops = if let Some(backup_path) = backup_dir_path { - check_dir_operations(&backup_path).is_ok() - } else { - true - }; - check_dir_operations(&dir_path).map(|_| check_backup_ops) - } - }) - .await - } - - async fn get_nodes_addresses(&self) -> Result { - let path = self.nodes_addresses_path(); - if !path.exists() { - return Ok(HashMap::new()); - } - async_blocking(move || { - let file = fs::File::open(path)?; - let reader = BufReader::new(file); - let nodes_addresses: HashMap = - serde_json::from_reader(reader).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - nodes_addresses - .iter() - .map(|(pubkey_str, addr)| { - let pubkey = PublicKey::from_str(pubkey_str) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - Ok((pubkey, *addr)) - }) - .collect() - }) - .await - } - - async fn save_nodes_addresses(&self, nodes_addresses: NodesAddressesMapShared) -> Result<(), Self::Error> { - let path = self.nodes_addresses_path(); - let backup_path = self.nodes_addresses_backup_path(); - async_blocking(move || { - let nodes_addresses: HashMap = nodes_addresses - .lock() - .iter() - .map(|(pubkey, addr)| (pubkey.to_string(), *addr)) - .collect(); - - let file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - serde_json::to_writer(file, &nodes_addresses) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - - if let Some(path) = backup_path { - let file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - serde_json::to_writer(file, &nodes_addresses) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - } - - Ok(()) - }) - .await - } - - async fn get_network_graph(&self, network: Network) -> Result { - let path = self.network_graph_path(); - if !path.exists() { - return Ok(NetworkGraph::new(genesis_block(network).header.block_hash())); - } - async_blocking(move || { - let file = fs::File::open(path)?; - common::log::info!("Reading the saved lightning network graph from file, this can take some time!"); - NetworkGraph::read(&mut BufReader::new(file)) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) - }) - .await - } - - async fn save_network_graph(&self, network_graph: Arc) -> Result<(), Self::Error> { - let path = self.network_graph_path(); - async_blocking(move || { - let file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - network_graph.write(&mut BufWriter::new(file)) - }) - .await - } - - async fn get_scorer(&self, network_graph: Arc) -> Result { - let path = self.scorer_path(); - if !path.exists() { - return Ok(Scorer::new(ProbabilisticScoringParameters::default(), network_graph)); - } - async_blocking(move || { - let file = fs::File::open(path)?; - Scorer::read( - &mut BufReader::new(file), - (ProbabilisticScoringParameters::default(), network_graph), - ) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) - }) - .await - } - - async fn save_scorer(&self, scorer: Arc>) -> Result<(), Self::Error> { - let path = self.scorer_path(); - async_blocking(move || { - let scorer = scorer.lock().unwrap(); - let file = fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - scorer.write(&mut BufWriter::new(file)) - }) - .await - } -} - -#[async_trait] -impl DbStorage for LightningPersister { - type Error = SqlError; - - async fn init_db(&self) -> Result<(), Self::Error> { - let sqlite_connection = self.sqlite_connection.clone(); - let sql_channels_history = create_channels_history_table_sql(self.storage_ticker.as_str())?; - let sql_payments_history = create_payments_history_table_sql(self.storage_ticker.as_str())?; - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - conn.execute(&sql_channels_history, NO_PARAMS).map(|_| ())?; - conn.execute(&sql_payments_history, NO_PARAMS).map(|_| ())?; - Ok(()) - }) - .await - } - - async fn is_db_initialized(&self) -> Result { - let channels_history_table = channels_history_table(self.storage_ticker.as_str()); - validate_table_name(&channels_history_table)?; - let payments_history_table = payments_history_table(self.storage_ticker.as_str()); - validate_table_name(&payments_history_table)?; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - let channels_history_initialized = - query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [channels_history_table], string_from_row)?; - let payments_history_initialized = - query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [payments_history_table], string_from_row)?; - Ok(channels_history_initialized.is_some() && payments_history_initialized.is_some()) - }) - .await - } - - async fn get_last_channel_rpc_id(&self) -> Result { - let sql = get_last_channel_rpc_id_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - let count: u32 = conn.query_row(&sql, NO_PARAMS, |r| r.get(0))?; - Ok(count) - }) - .await - } - - async fn add_channel_to_db(&self, details: SqlChannelDetails) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let rpc_id = details.rpc_id.to_string(); - let channel_id = details.channel_id; - let counterparty_node_id = details.counterparty_node_id; - let is_outbound = (details.is_outbound as i32).to_string(); - let is_public = (details.is_public as i32).to_string(); - let is_closed = (details.is_closed as i32).to_string(); - let created_at = (details.created_at as u32).to_string(); - - let params = [ - rpc_id, - channel_id, - counterparty_node_id, - is_outbound, - is_public, - is_closed, - created_at, - ]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&insert_channel_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn add_funding_tx_to_db( - &self, - rpc_id: u64, - funding_tx: String, - funding_value: u64, - funding_generated_in_block: u64, - ) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let funding_value = funding_value.to_string(); - let funding_generated_in_block = funding_generated_in_block.to_string(); - let rpc_id = rpc_id.to_string(); - - let params = [funding_tx, funding_value, funding_generated_in_block, rpc_id]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_funding_tx_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn update_funding_tx_block_height(&self, funding_tx: String, block_height: u64) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let generated_in_block = block_height as u32; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - let params = [&generated_in_block as &dyn ToSql, &funding_tx as &dyn ToSql]; - sql_transaction.execute(&update_funding_tx_block_height_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn update_channel_to_closed( - &self, - rpc_id: u64, - closure_reason: String, - closed_at: u64, - ) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let is_closed = "1".to_string(); - let rpc_id = rpc_id.to_string(); - - let params = [closure_reason, is_closed, closed_at.to_string(), rpc_id]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_channel_to_closed_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn get_closed_channels_with_no_closing_tx(&self) -> Result, Self::Error> { - let mut builder = get_channels_builder_preimage(self.storage_ticker.as_str())?; - builder.and_where("closing_tx IS NULL"); - add_fields_to_get_channels_sql_builder(&mut builder); - let sql = builder.sql().expect("valid sql"); - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - - let mut stmt = conn.prepare(&sql)?; - let result = stmt - .query_map_named(&[], channel_details_from_row)? - .collect::>()?; - Ok(result) - }) - .await - } - - async fn add_closing_tx_to_db(&self, rpc_id: u64, closing_tx: String) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let rpc_id = rpc_id.to_string(); - - let params = [closing_tx, rpc_id]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_closing_tx_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn add_claiming_tx_to_db( - &self, - closing_tx: String, - claiming_tx: String, - claimed_balance: f64, - ) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let claimed_balance = claimed_balance.to_string(); - - let params = [claiming_tx, claimed_balance, closing_tx]; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&update_claiming_tx_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn get_channel_from_db(&self, rpc_id: u64) -> Result, Self::Error> { - let params = [rpc_id.to_string()]; - let sql = select_channel_by_rpc_id_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - query_single_row(&conn, &sql, params, channel_details_from_row) - }) - .await - } - - async fn get_closed_channels_by_filter( - &self, - filter: Option, - paging: PagingOptionsEnum, - limit: usize, - ) -> Result { - let mut sql_builder = get_channels_builder_preimage(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - - let mut total_builder = sql_builder.clone(); - total_builder.count("id"); - let total_sql = total_builder.sql().expect("valid sql"); - let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; - let total = total.try_into().expect("count should be always above zero"); - - let offset = match paging { - PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, - PagingOptionsEnum::FromId(rpc_id) => { - let params = [rpc_id as u32]; - let maybe_offset = - offset_by_id(&conn, &sql_builder, params, "rpc_id", "closed_at DESC", "rpc_id = ?1")?; - match maybe_offset { - Some(offset) => offset, - None => { - return Ok(GetClosedChannelsResult { - channels: vec![], - skipped: 0, - total, - }) - }, - } - }, - }; - - let mut params = vec![]; - if let Some(f) = filter { - apply_get_channels_filter(&mut sql_builder, &mut params, f); - } - let params_as_trait: Vec<_> = params.iter().map(|(key, value)| (*key, value as &dyn ToSql)).collect(); - add_fields_to_get_channels_sql_builder(&mut sql_builder); - finalize_get_channels_sql_builder(&mut sql_builder, offset, limit); - - let sql = sql_builder.sql().expect("valid sql"); - let mut stmt = conn.prepare(&sql)?; - let channels = stmt - .query_map_named(params_as_trait.as_slice(), channel_details_from_row)? - .collect::>()?; - let result = GetClosedChannelsResult { - channels, - skipped: offset, - total, - }; - Ok(result) - }) - .await - } - - async fn add_or_update_payment_in_db(&self, info: PaymentInfo) -> Result<(), Self::Error> { - let for_coin = self.storage_ticker.clone(); - let payment_hash = hex::encode(info.payment_hash.0); - let (is_outbound, destination) = match info.payment_type { - PaymentType::OutboundPayment { destination } => (true as i32, Some(destination.to_string())), - PaymentType::InboundPayment => (false as i32, None), - }; - let description = info.description; - let preimage = info.preimage.map(|p| hex::encode(p.0)); - let secret = info.secret.map(|s| hex::encode(s.0)); - let amount_msat = info.amt_msat.map(|a| a as u32); - let fee_paid_msat = info.fee_paid_msat.map(|f| f as u32); - let status = info.status.to_string(); - let created_at = info.created_at as u32; - let last_updated = info.last_updated as u32; - - let sqlite_connection = self.sqlite_connection.clone(); - async_blocking(move || { - let params = [ - &payment_hash as &dyn ToSql, - &destination as &dyn ToSql, - &description as &dyn ToSql, - &preimage as &dyn ToSql, - &secret as &dyn ToSql, - &amount_msat as &dyn ToSql, - &fee_paid_msat as &dyn ToSql, - &is_outbound as &dyn ToSql, - &status as &dyn ToSql, - &created_at as &dyn ToSql, - &last_updated as &dyn ToSql, - ]; - let mut conn = sqlite_connection.lock().unwrap(); - let sql_transaction = conn.transaction()?; - sql_transaction.execute(&upsert_payment_sql(&for_coin)?, ¶ms)?; - sql_transaction.commit()?; - Ok(()) - }) - .await - } - - async fn get_payment_from_db(&self, hash: PaymentHash) -> Result, Self::Error> { - let params = [hex::encode(hash.0)]; - let sql = select_payment_by_hash_sql(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - query_single_row(&conn, &sql, params, payment_info_from_row) - }) - .await - } - - async fn get_payments_by_filter( - &self, - filter: Option, - paging: PagingOptionsEnum, - limit: usize, - ) -> Result { - let mut sql_builder = get_payments_builder_preimage(self.storage_ticker.as_str())?; - let sqlite_connection = self.sqlite_connection.clone(); - - async_blocking(move || { - let conn = sqlite_connection.lock().unwrap(); - - let mut total_builder = sql_builder.clone(); - total_builder.count("id"); - let total_sql = total_builder.sql().expect("valid sql"); - let total: isize = conn.query_row(&total_sql, NO_PARAMS, |row| row.get(0))?; - let total = total.try_into().expect("count should be always above zero"); - - let offset = match paging { - PagingOptionsEnum::PageNumber(page) => (page.get() - 1) * limit, - PagingOptionsEnum::FromId(hash) => { - let hash_str = hex::encode(hash.0); - let params = [&hash_str]; - let maybe_offset = offset_by_id( - &conn, - &sql_builder, - params, - "payment_hash", - "last_updated DESC", - "payment_hash = ?1", - )?; - match maybe_offset { - Some(offset) => offset, - None => { - return Ok(GetPaymentsResult { - payments: vec![], - skipped: 0, - total, - }) - }, - } - }, - }; - - let mut params = vec![]; - if let Some(f) = filter { - apply_get_payments_filter(&mut sql_builder, &mut params, f); - } - let params_as_trait: Vec<_> = params.iter().map(|(key, value)| (*key, value as &dyn ToSql)).collect(); - finalize_get_payments_sql_builder(&mut sql_builder, offset, limit); - - let sql = sql_builder.sql().expect("valid sql"); - let mut stmt = conn.prepare(&sql)?; - let payments = stmt - .query_map_named(params_as_trait.as_slice(), payment_info_from_row)? - .collect::>()?; - let result = GetPaymentsResult { - payments, - skipped: offset, - total, - }; - Ok(result) - }) - .await - } -} - -#[cfg(test)] -mod tests { - use super::*; - extern crate bitcoin; - extern crate lightning; - use bitcoin::blockdata::block::{Block, BlockHeader}; - use bitcoin::hashes::hex::FromHex; - use bitcoin::Txid; - use common::{block_on, now_ms}; - use db_common::sqlite::rusqlite::Connection; - use lightning::chain::chainmonitor::Persist; - use lightning::chain::transaction::OutPoint; - use lightning::chain::ChannelMonitorUpdateErr; - use lightning::ln::features::InitFeatures; - use lightning::ln::functional_test_utils::*; - use lightning::util::events::{ClosureReason, MessageSendEventsProvider}; - use lightning::util::test_utils; - use lightning::{check_added_monitors, check_closed_broadcast, check_closed_event}; - use rand::distributions::Alphanumeric; - use rand::{Rng, RngCore}; - use secp256k1::{Secp256k1, SecretKey}; - use std::fs; - use std::num::NonZeroUsize; - use std::path::PathBuf; - use std::sync::{Arc, Mutex}; - - impl Drop for LightningPersister { - fn drop(&mut self) { - // We test for invalid directory names, so it's OK if directory removal - // fails. - match fs::remove_dir_all(&self.main_path) { - Err(e) => println!("Failed to remove test persister directory: {}", e), - _ => {}, - } - } - } - - fn generate_random_channels(num: u64) -> Vec { - let mut rng = rand::thread_rng(); - let mut channels = vec![]; - let s = Secp256k1::new(); - let mut bytes = [0; 32]; - for i in 0..num { - let details = SqlChannelDetails { - rpc_id: i + 1, - channel_id: { - rng.fill_bytes(&mut bytes); - hex::encode(bytes) - }, - counterparty_node_id: { - rng.fill_bytes(&mut bytes); - let secret = SecretKey::from_slice(&bytes).unwrap(); - let pubkey = PublicKey::from_secret_key(&s, &secret); - pubkey.to_string() - }, - funding_tx: { - rng.fill_bytes(&mut bytes); - Some(hex::encode(bytes)) - }, - funding_value: Some(rng.gen::() as u64), - closing_tx: { - rng.fill_bytes(&mut bytes); - Some(hex::encode(bytes)) - }, - closure_reason: { - Some( - rng.sample_iter(&Alphanumeric) - .take(30) - .map(char::from) - .collect::(), - ) - }, - claiming_tx: { - rng.fill_bytes(&mut bytes); - Some(hex::encode(bytes)) - }, - claimed_balance: Some(rng.gen::()), - funding_generated_in_block: Some(rng.gen::() as u64), - is_outbound: rand::random(), - is_public: rand::random(), - is_closed: rand::random(), - created_at: rng.gen::() as u64, - closed_at: Some(rng.gen::() as u64), - }; - channels.push(details); - } - channels - } - - fn generate_random_payments(num: u64) -> Vec { - let mut rng = rand::thread_rng(); - let mut payments = vec![]; - let s = Secp256k1::new(); - let mut bytes = [0; 32]; - for _ in 0..num { - let payment_type = if let 0 = rng.gen::() % 2 { - PaymentType::InboundPayment - } else { - rng.fill_bytes(&mut bytes); - let secret = SecretKey::from_slice(&bytes).unwrap(); - PaymentType::OutboundPayment { - destination: PublicKey::from_secret_key(&s, &secret), - } - }; - let status_rng: u8 = rng.gen(); - let status = if status_rng % 3 == 0 { - HTLCStatus::Succeeded - } else if status_rng % 3 == 1 { - HTLCStatus::Pending - } else { - HTLCStatus::Failed - }; - let description: String = rng.sample_iter(&Alphanumeric).take(30).map(char::from).collect(); - let info = PaymentInfo { - payment_hash: { - rng.fill_bytes(&mut bytes); - PaymentHash(bytes) - }, - payment_type, - description, - preimage: { - rng.fill_bytes(&mut bytes); - Some(PaymentPreimage(bytes)) - }, - secret: { - rng.fill_bytes(&mut bytes); - Some(PaymentSecret(bytes)) - }, - amt_msat: Some(rng.gen::() as u64), - fee_paid_msat: Some(rng.gen::() as u64), - status, - created_at: rng.gen::() as u64, - last_updated: rng.gen::() as u64, - }; - payments.push(info); - } - payments - } - - // Integration-test the LightningPersister. Test relaying a few payments - // and check that the persisted data is updated the appropriate number of - // times. - #[test] - fn test_filesystem_persister() { - // Create the nodes, giving them LightningPersisters for data persisters. - let persister_0 = LightningPersister::new( - "test_filesystem_persister_0".into(), - PathBuf::from("test_filesystem_persister_0"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - let persister_1 = LightningPersister::new( - "test_filesystem_persister_1".into(), - PathBuf::from("test_filesystem_persister_1"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - let chanmon_cfgs = create_chanmon_cfgs(2); - let mut node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let chain_mon_0 = test_utils::TestChainMonitor::new( - Some(&chanmon_cfgs[0].chain_source), - &chanmon_cfgs[0].tx_broadcaster, - &chanmon_cfgs[0].logger, - &chanmon_cfgs[0].fee_estimator, - &persister_0, - &node_cfgs[0].keys_manager, - ); - let chain_mon_1 = test_utils::TestChainMonitor::new( - Some(&chanmon_cfgs[1].chain_source), - &chanmon_cfgs[1].tx_broadcaster, - &chanmon_cfgs[1].logger, - &chanmon_cfgs[1].fee_estimator, - &persister_1, - &node_cfgs[1].keys_manager, - ); - node_cfgs[0].chain_monitor = chain_mon_0; - node_cfgs[1].chain_monitor = chain_mon_1; - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - - // Check that the persisted channel data is empty before any channels are - // open. - let mut persisted_chan_data_0 = persister_0.read_channelmonitors(nodes[0].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_0.len(), 0); - let mut persisted_chan_data_1 = persister_1.read_channelmonitors(nodes[1].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_1.len(), 0); - - // Helper to make sure the channel is on the expected update ID. - macro_rules! check_persisted_data { - ($expected_update_id: expr) => { - persisted_chan_data_0 = persister_0.read_channelmonitors(nodes[0].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_0.len(), 1); - for (_, mon) in persisted_chan_data_0.iter() { - assert_eq!(mon.get_latest_update_id(), $expected_update_id); - } - persisted_chan_data_1 = persister_1.read_channelmonitors(nodes[1].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_1.len(), 1); - for (_, mon) in persisted_chan_data_1.iter() { - assert_eq!(mon.get_latest_update_id(), $expected_update_id); - } - }; - } - - // Create some initial channel and check that a channel was persisted. - let _ = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - check_persisted_data!(0); - - // Send a few payments and make sure the monitors are updated to the latest. - send_payment(&nodes[0], &vec![&nodes[1]][..], 8000000); - check_persisted_data!(5); - send_payment(&nodes[1], &vec![&nodes[0]][..], 4000000); - check_persisted_data!(10); - - // Force close because cooperative close doesn't result in any persisted - // updates. - nodes[0] - .node - .force_close_channel(&nodes[0].node.list_channels()[0].channel_id) - .unwrap(); - check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed); - check_closed_broadcast!(nodes[0], true); - check_added_monitors!(nodes[0], 1); - - let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap(); - assert_eq!(node_txn.len(), 1); - - let header = BlockHeader { - version: 0x20000000, - prev_blockhash: nodes[0].best_block_hash(), - merkle_root: Default::default(), - time: 42, - bits: 42, - nonce: 42, - }; - connect_block(&nodes[1], &Block { - header, - txdata: vec![node_txn[0].clone(), node_txn[0].clone()], - }); - check_closed_broadcast!(nodes[1], true); - check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed); - check_added_monitors!(nodes[1], 1); - - // Make sure everything is persisted as expected after close. - check_persisted_data!(11); - } - - // Test that if the persister's path to channel data is read-only, writing a - // monitor to it results in the persister returning a PermanentFailure. - // Windows ignores the read-only flag for folders, so this test is Unix-only. - #[cfg(not(target_os = "windows"))] - #[test] - fn test_readonly_dir_perm_failure() { - let persister = LightningPersister::new( - "test_readonly_dir_perm_failure".into(), - PathBuf::from("test_readonly_dir_perm_failure"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - fs::create_dir_all(&persister.main_path).unwrap(); - - // Set up a dummy channel and force close. This will produce a monitor - // that we can then use to test persistence. - let chanmon_cfgs = create_chanmon_cfgs(2); - let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - let chan = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - nodes[1].node.force_close_channel(&chan.2).unwrap(); - check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed); - let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap(); - let update_map = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap(); - let update_id = update_map.get(&added_monitors[0].0.to_channel_id()).unwrap(); - - // Set the persister's directory to read-only, which should result in - // returning a permanent failure when we then attempt to persist a - // channel update. - let path = &persister.main_path; - let mut perms = fs::metadata(path).unwrap().permissions(); - perms.set_readonly(true); - fs::set_permissions(path, perms).unwrap(); - - let test_txo = OutPoint { - txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), - index: 0, - }; - match persister.persist_new_channel(test_txo, &added_monitors[0].1, update_id.2) { - Err(ChannelMonitorUpdateErr::PermanentFailure) => {}, - _ => panic!("unexpected result from persisting new channel"), - } - - nodes[1].node.get_and_clear_pending_msg_events(); - added_monitors.clear(); - } - - // Test that if a persister's directory name is invalid, monitor persistence - // will fail. - #[cfg(target_os = "windows")] - #[test] - fn test_fail_on_open() { - // Set up a dummy channel and force close. This will produce a monitor - // that we can then use to test persistence. - let chanmon_cfgs = create_chanmon_cfgs(2); - let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - let chan = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()); - nodes[1].node.force_close_channel(&chan.2).unwrap(); - check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed); - let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap(); - let update_map = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap(); - let update_id = update_map.get(&added_monitors[0].0.to_channel_id()).unwrap(); - - // Create the persister with an invalid directory name and test that the - // channel fails to open because the directories fail to be created. There - // don't seem to be invalid filename characters on Unix that Rust doesn't - // handle, hence why the test is Windows-only. - let persister = LightningPersister::new( - "test_fail_on_open".into(), - PathBuf::from(":<>/"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - let test_txo = OutPoint { - txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), - index: 0, - }; - match persister.persist_new_channel(test_txo, &added_monitors[0].1, update_id.2) { - Err(ChannelMonitorUpdateErr::PermanentFailure) => {}, - _ => panic!("unexpected result from persisting new channel"), - } - - nodes[1].node.get_and_clear_pending_msg_events(); - added_monitors.clear(); - } - - #[test] - fn test_init_sql_collection() { - let persister = LightningPersister::new( - "init_sql_collection".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - let initialized = block_on(persister.is_db_initialized()).unwrap(); - assert!(!initialized); - - block_on(persister.init_db()).unwrap(); - // repetitive init must not fail - block_on(persister.init_db()).unwrap(); - - let initialized = block_on(persister.is_db_initialized()).unwrap(); - assert!(initialized); - } - - #[test] - fn test_add_get_channel_sql() { - let persister = LightningPersister::new( - "add_get_channel".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - block_on(persister.init_db()).unwrap(); - - let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); - assert_eq!(last_channel_rpc_id, 0); - - let channel = block_on(persister.get_channel_from_db(1)).unwrap(); - assert!(channel.is_none()); - - let mut expected_channel_details = SqlChannelDetails::new( - 1, - [0; 32], - PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9").unwrap(), - true, - true, - ); - block_on(persister.add_channel_to_db(expected_channel_details.clone())).unwrap(); - let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); - assert_eq!(last_channel_rpc_id, 1); - - let actual_channel_details = block_on(persister.get_channel_from_db(1)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - // must fail because we are adding channel with the same rpc_id - block_on(persister.add_channel_to_db(expected_channel_details.clone())).unwrap_err(); - assert_eq!(last_channel_rpc_id, 1); - - expected_channel_details.rpc_id = 2; - block_on(persister.add_channel_to_db(expected_channel_details.clone())).unwrap(); - let last_channel_rpc_id = block_on(persister.get_last_channel_rpc_id()).unwrap(); - assert_eq!(last_channel_rpc_id, 2); - - block_on(persister.add_funding_tx_to_db( - 2, - "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), - 3000, - 50000, - )) - .unwrap(); - expected_channel_details.funding_tx = - Some("9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into()); - expected_channel_details.funding_value = Some(3000); - expected_channel_details.funding_generated_in_block = Some(50000); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - block_on(persister.update_funding_tx_block_height( - "9cdafd6d42dcbdc06b0b5bce1866deb82630581285bbfb56870577300c0a8c6e".into(), - 50001, - )) - .unwrap(); - expected_channel_details.funding_generated_in_block = Some(50001); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - let current_time = now_ms() / 1000; - block_on(persister.update_channel_to_closed(2, "the channel was cooperatively closed".into(), current_time)) - .unwrap(); - expected_channel_details.closure_reason = Some("the channel was cooperatively closed".into()); - expected_channel_details.is_closed = true; - expected_channel_details.closed_at = Some(current_time); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); - assert_eq!(actual_channels.len(), 1); - - let closed_channels = - block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); - assert_eq!(closed_channels.channels.len(), 1); - assert_eq!(expected_channel_details, closed_channels.channels[0]); - - block_on(persister.update_channel_to_closed(1, "the channel was cooperatively closed".into(), now_ms() / 1000)) - .unwrap(); - let closed_channels = - block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 10)).unwrap(); - assert_eq!(closed_channels.channels.len(), 2); - - let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); - assert_eq!(actual_channels.len(), 2); - - block_on(persister.add_closing_tx_to_db( - 2, - "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), - )) - .unwrap(); - expected_channel_details.closing_tx = - Some("5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into()); - - let actual_channels = block_on(persister.get_closed_channels_with_no_closing_tx()).unwrap(); - assert_eq!(actual_channels.len(), 1); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - - block_on(persister.add_claiming_tx_to_db( - "5557df9ad2c9b3c57a4df8b4a7da0b7a6f4e923b4a01daa98bf9e5a3b33e9c8f".into(), - "97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into(), - 2000.333333, - )) - .unwrap(); - expected_channel_details.claiming_tx = - Some("97f061634a4a7b0b0c2b95648f86b1c39b95e0cf5073f07725b7143c095b612a".into()); - expected_channel_details.claimed_balance = Some(2000.333333); - - let actual_channel_details = block_on(persister.get_channel_from_db(2)).unwrap().unwrap(); - assert_eq!(expected_channel_details, actual_channel_details); - } - - #[test] - fn test_add_get_payment_sql() { - let persister = LightningPersister::new( - "add_get_payment".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - block_on(persister.init_db()).unwrap(); - - let payment = block_on(persister.get_payment_from_db(PaymentHash([0; 32]))).unwrap(); - assert!(payment.is_none()); - - let mut expected_payment_info = PaymentInfo { - payment_hash: PaymentHash([0; 32]), - payment_type: PaymentType::InboundPayment, - description: "test payment".into(), - preimage: Some(PaymentPreimage([2; 32])), - secret: Some(PaymentSecret([3; 32])), - amt_msat: Some(2000), - fee_paid_msat: Some(100), - status: HTLCStatus::Failed, - created_at: now_ms() / 1000, - last_updated: now_ms() / 1000, - }; - block_on(persister.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); - - let actual_payment_info = block_on(persister.get_payment_from_db(PaymentHash([0; 32]))) - .unwrap() - .unwrap(); - assert_eq!(expected_payment_info, actual_payment_info); - - expected_payment_info.payment_hash = PaymentHash([1; 32]); - expected_payment_info.payment_type = PaymentType::OutboundPayment { - destination: PublicKey::from_str("038863cf8ab91046230f561cd5b386cbff8309fa02e3f0c3ed161a3aeb64a643b9") - .unwrap(), - }; - expected_payment_info.secret = None; - expected_payment_info.amt_msat = None; - expected_payment_info.status = HTLCStatus::Succeeded; - expected_payment_info.last_updated = now_ms() / 1000; - block_on(persister.add_or_update_payment_in_db(expected_payment_info.clone())).unwrap(); - - let actual_payment_info = block_on(persister.get_payment_from_db(PaymentHash([1; 32]))) - .unwrap() - .unwrap(); - assert_eq!(expected_payment_info, actual_payment_info); - } - - #[test] - fn test_get_payments_by_filter() { - let persister = LightningPersister::new( - "test_get_payments_by_filter".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - block_on(persister.init_db()).unwrap(); - - let mut payments = generate_random_payments(100); - - for payment in payments.clone() { - block_on(persister.add_or_update_payment_in_db(payment)).unwrap(); - } - - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); - let limit = 4; - - let result = block_on(persister.get_payments_by_filter(None, paging, limit)).unwrap(); - - payments.sort_by(|a, b| b.last_updated.cmp(&a.last_updated)); - let expected_payments = &payments[..4].to_vec(); - let actual_payments = &result.payments; - - assert_eq!(0, result.skipped); - assert_eq!(100, result.total); - assert_eq!(expected_payments, actual_payments); - - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); - let limit = 5; - - let result = block_on(persister.get_payments_by_filter(None, paging, limit)).unwrap(); - - let expected_payments = &payments[5..10].to_vec(); - let actual_payments = &result.payments; - - assert_eq!(5, result.skipped); - assert_eq!(100, result.total); - assert_eq!(expected_payments, actual_payments); - - let from_payment_hash = payments[20].payment_hash; - let paging = PagingOptionsEnum::FromId(from_payment_hash); - let limit = 3; - - let result = block_on(persister.get_payments_by_filter(None, paging, limit)).unwrap(); - - let expected_payments = &payments[21..24].to_vec(); - let actual_payments = &result.payments; - - assert_eq!(expected_payments, actual_payments); - - let mut filter = PaymentsFilter { - payment_type: Some(PaymentType::InboundPayment), - description: None, - status: None, - from_amount_msat: None, - to_amount_msat: None, - from_fee_paid_msat: None, - to_fee_paid_msat: None, - from_timestamp: None, - to_timestamp: None, - }; - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); - let limit = 10; - - let result = block_on(persister.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); - let expected_payments_vec: Vec = payments - .iter() - .map(|p| p.clone()) - .filter(|p| p.payment_type == PaymentType::InboundPayment) - .collect(); - let expected_payments = if expected_payments_vec.len() > 10 { - expected_payments_vec[..10].to_vec() - } else { - expected_payments_vec.clone() - }; - let actual_payments = result.payments; - - assert_eq!(expected_payments, actual_payments); - - filter.status = Some(HTLCStatus::Succeeded); - let result = block_on(persister.get_payments_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); - let expected_payments_vec: Vec = expected_payments_vec - .iter() - .map(|p| p.clone()) - .filter(|p| p.status == HTLCStatus::Succeeded) - .collect(); - let expected_payments = if expected_payments_vec.len() > 10 { - expected_payments_vec[..10].to_vec() - } else { - expected_payments_vec - }; - let actual_payments = result.payments; - - assert_eq!(expected_payments, actual_payments); - - let description = &payments[42].description; - let substr = &description[5..10]; - filter.payment_type = None; - filter.status = None; - filter.description = Some(substr.to_string()); - let result = block_on(persister.get_payments_by_filter(Some(filter), paging, limit)).unwrap(); - let expected_payments_vec: Vec = payments - .iter() - .map(|p| p.clone()) - .filter(|p| p.description.contains(&substr)) - .collect(); - let expected_payments = if expected_payments_vec.len() > 10 { - expected_payments_vec[..10].to_vec() - } else { - expected_payments_vec.clone() - }; - let actual_payments = result.payments; - - assert_eq!(expected_payments, actual_payments); - } - - #[test] - fn test_get_channels_by_filter() { - let persister = LightningPersister::new( - "test_get_channels_by_filter".into(), - PathBuf::from("test_filesystem_persister"), - None, - Arc::new(Mutex::new(Connection::open_in_memory().unwrap())), - ); - - block_on(persister.init_db()).unwrap(); - - let channels = generate_random_channels(100); - - for channel in channels { - block_on(persister.add_channel_to_db(channel.clone())).unwrap(); - block_on(persister.add_funding_tx_to_db( - channel.rpc_id, - channel.funding_tx.unwrap(), - channel.funding_value.unwrap(), - channel.funding_generated_in_block.unwrap(), - )) - .unwrap(); - block_on(persister.update_channel_to_closed(channel.rpc_id, channel.closure_reason.unwrap(), 1655806080)) - .unwrap(); - block_on(persister.add_closing_tx_to_db(channel.rpc_id, channel.closing_tx.clone().unwrap())).unwrap(); - block_on(persister.add_claiming_tx_to_db( - channel.closing_tx.unwrap(), - channel.claiming_tx.unwrap(), - channel.claimed_balance.unwrap(), - )) - .unwrap(); - } - - // get all channels from SQL since updated_at changed from channels generated by generate_random_channels - let channels = block_on(persister.get_closed_channels_by_filter(None, PagingOptionsEnum::default(), 100)) - .unwrap() - .channels; - assert_eq!(100, channels.len()); - - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); - let limit = 4; - - let result = block_on(persister.get_closed_channels_by_filter(None, paging, limit)).unwrap(); - - let expected_channels = &channels[..4].to_vec(); - let actual_channels = &result.channels; - - assert_eq!(0, result.skipped); - assert_eq!(100, result.total); - assert_eq!(expected_channels, actual_channels); - - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(2).unwrap()); - let limit = 5; - - let result = block_on(persister.get_closed_channels_by_filter(None, paging, limit)).unwrap(); - - let expected_channels = &channels[5..10].to_vec(); - let actual_channels = &result.channels; - - assert_eq!(5, result.skipped); - assert_eq!(100, result.total); - assert_eq!(expected_channels, actual_channels); - - let from_rpc_id = 20; - let paging = PagingOptionsEnum::FromId(from_rpc_id); - let limit = 3; - - let result = block_on(persister.get_closed_channels_by_filter(None, paging, limit)).unwrap(); - - let expected_channels = channels[20..23].to_vec(); - let actual_channels = result.channels; - - assert_eq!(expected_channels, actual_channels); - - let mut filter = ClosedChannelsFilter { - channel_id: None, - counterparty_node_id: None, - funding_tx: None, - from_funding_value: None, - to_funding_value: None, - closing_tx: None, - closure_reason: None, - claiming_tx: None, - from_claimed_balance: None, - to_claimed_balance: None, - channel_type: Some(ChannelType::Outbound), - channel_visibility: None, - }; - let paging = PagingOptionsEnum::PageNumber(NonZeroUsize::new(1).unwrap()); - let limit = 10; - - let result = - block_on(persister.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); - let expected_channels_vec: Vec = channels - .iter() - .map(|chan| chan.clone()) - .filter(|chan| chan.is_outbound) - .collect(); - let expected_channels = if expected_channels_vec.len() > 10 { - expected_channels_vec[..10].to_vec() - } else { - expected_channels_vec.clone() - }; - let actual_channels = result.channels; - - assert_eq!(expected_channels, actual_channels); - - filter.channel_visibility = Some(ChannelVisibility::Public); - let result = - block_on(persister.get_closed_channels_by_filter(Some(filter.clone()), paging.clone(), limit)).unwrap(); - let expected_channels_vec: Vec = expected_channels_vec - .iter() - .map(|chan| chan.clone()) - .filter(|chan| chan.is_public) - .collect(); - let expected_channels = if expected_channels_vec.len() > 10 { - expected_channels_vec[..10].to_vec() - } else { - expected_channels_vec - }; - let actual_channels = result.channels; - - assert_eq!(expected_channels, actual_channels); - - let channel_id = channels[42].channel_id.clone(); - filter.channel_type = None; - filter.channel_visibility = None; - filter.channel_id = Some(channel_id.clone()); - let result = block_on(persister.get_closed_channels_by_filter(Some(filter), paging, limit)).unwrap(); - let expected_channels_vec: Vec = channels - .iter() - .map(|chan| chan.clone()) - .filter(|chan| chan.channel_id == channel_id) - .collect(); - let expected_channels = if expected_channels_vec.len() > 10 { - expected_channels_vec[..10].to_vec() - } else { - expected_channels_vec.clone() - }; - let actual_channels = result.channels; - - assert_eq!(expected_channels, actual_channels); - } -} diff --git a/mm2src/coins/lightning_persister/src/util.rs b/mm2src/coins/lightning_persister/src/util.rs deleted file mode 100644 index ac5bc99de5..0000000000 --- a/mm2src/coins/lightning_persister/src/util.rs +++ /dev/null @@ -1,196 +0,0 @@ -#[cfg(target_os = "windows")] extern crate winapi; - -use std::fs; -use std::path::{Path, PathBuf}; - -#[cfg(not(target_os = "windows"))] -use std::os::unix::io::AsRawFd; - -#[cfg(target_os = "windows")] -use {std::ffi::OsStr, std::os::windows::ffi::OsStrExt}; - -pub(crate) trait DiskWriteable { - fn write_to_file(&self, writer: &mut fs::File) -> Result<(), std::io::Error>; -} - -pub(crate) fn get_full_filepath(mut filepath: PathBuf, filename: String) -> String { - filepath.push(filename); - filepath.to_str().unwrap().to_string() -} - -#[cfg(target_os = "windows")] -macro_rules! call { - ($e: expr) => { - if $e != 0 { - return Ok(()); - } else { - return Err(std::io::Error::last_os_error()); - } - }; -} - -#[cfg(target_os = "windows")] -fn path_to_windows_str>(path: T) -> Vec { - path.as_ref().encode_wide().chain(Some(0)).collect() -} - -#[allow(bare_trait_objects)] -pub(crate) fn write_to_file(path: PathBuf, filename: String, data: &D) -> std::io::Result<()> { - fs::create_dir_all(path.clone())?; - // Do a crazy dance with lots of fsync()s to be overly cautious here... - // We never want to end up in a state where we've lost the old data, or end up using the - // old data on power loss after we've returned. - // The way to atomically write a file on Unix platforms is: - // open(tmpname), write(tmpfile), fsync(tmpfile), close(tmpfile), rename(), fsync(dir) - let filename_with_path = get_full_filepath(path, filename); - let tmp_filename = format!("{}.tmp", filename_with_path); - - { - // Note that going by rust-lang/rust@d602a6b, on MacOS it is only safe to use - // rust stdlib 1.36 or higher. - let mut f = fs::File::create(&tmp_filename)?; - data.write_to_file(&mut f)?; - f.sync_all()?; - } - // Fsync the parent directory on Unix. - #[cfg(not(target_os = "windows"))] - { - fs::rename(&tmp_filename, &filename_with_path)?; - let path = Path::new(&filename_with_path).parent().unwrap(); - let dir_file = fs::OpenOptions::new().read(true).open(path)?; - unsafe { - libc::fsync(dir_file.as_raw_fd()); - } - } - #[cfg(target_os = "windows")] - { - let src = PathBuf::from(tmp_filename); - let dst = PathBuf::from(filename_with_path.clone()); - if Path::new(&filename_with_path).exists() { - unsafe { - winapi::um::winbase::ReplaceFileW( - path_to_windows_str(dst).as_ptr(), - path_to_windows_str(src).as_ptr(), - std::ptr::null(), - winapi::um::winbase::REPLACEFILE_IGNORE_MERGE_ERRORS, - std::ptr::null_mut() as *mut winapi::ctypes::c_void, - std::ptr::null_mut() as *mut winapi::ctypes::c_void, - ) - }; - } else { - call!(unsafe { - winapi::um::winbase::MoveFileExW( - path_to_windows_str(src).as_ptr(), - path_to_windows_str(dst).as_ptr(), - winapi::um::winbase::MOVEFILE_WRITE_THROUGH | winapi::um::winbase::MOVEFILE_REPLACE_EXISTING, - ) - }); - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::{get_full_filepath, write_to_file, DiskWriteable}; - use std::fs; - use std::io; - use std::io::Write; - use std::path::PathBuf; - - struct TestWriteable {} - impl DiskWriteable for TestWriteable { - fn write_to_file(&self, writer: &mut fs::File) -> Result<(), io::Error> { writer.write_all(&[42; 1]) } - } - - // Test that if the persister's path to channel data is read-only, writing - // data to it fails. Windows ignores the read-only flag for folders, so this - // test is Unix-only. - #[cfg(not(target_os = "windows"))] - #[test] - fn test_readonly_dir() { - let test_writeable = TestWriteable {}; - let filename = "test_readonly_dir_persister_filename".to_string(); - let path = "test_readonly_dir_persister_dir"; - fs::create_dir_all(path.to_string()).unwrap(); - let mut perms = fs::metadata(path.to_string()).unwrap().permissions(); - perms.set_readonly(true); - fs::set_permissions(path.to_string(), perms).unwrap(); - match write_to_file(PathBuf::from(path.to_string()), filename, &test_writeable) { - Err(e) => assert_eq!(e.kind(), io::ErrorKind::PermissionDenied), - _ => panic!("Unexpected error message"), - } - let mut perms = fs::metadata(path.to_string()).unwrap().permissions(); - perms.set_readonly(false); - fs::set_permissions(path.to_string(), perms).unwrap(); - fs::remove_dir_all(path).unwrap(); - } - - // Test failure to rename in the process of atomically creating a channel - // monitor's file. We induce this failure by making the `tmp` file a - // directory. - // Explanation: given "from" = the file being renamed, "to" = the destination - // file that already exists: Unix should fail because if "from" is a file, - // then "to" is also required to be a file. - // TODO: ideally try to make this work on Windows again - #[cfg(not(target_os = "windows"))] - #[test] - fn test_rename_failure() { - let test_writeable = TestWriteable {}; - let filename = "test_rename_failure_filename"; - let path = PathBuf::from("test_rename_failure_dir"); - // Create the channel data file and make it a directory. - fs::create_dir_all(get_full_filepath(path.clone(), filename.to_string())).unwrap(); - match write_to_file(path.clone(), filename.to_string(), &test_writeable) { - Err(e) => assert_eq!(e.raw_os_error(), Some(libc::EISDIR)), - _ => panic!("Unexpected Ok(())"), - } - fs::remove_dir_all(path).unwrap(); - } - - #[test] - fn test_diskwriteable_failure() { - struct FailingWriteable {} - impl DiskWriteable for FailingWriteable { - fn write_to_file(&self, _writer: &mut fs::File) -> Result<(), std::io::Error> { - Err(std::io::Error::new(std::io::ErrorKind::Other, "expected failure")) - } - } - - let filename = "test_diskwriteable_failure"; - let path = PathBuf::from("test_diskwriteable_failure_dir"); - let test_writeable = FailingWriteable {}; - match write_to_file(path.clone(), filename.to_string(), &test_writeable) { - Err(e) => { - assert_eq!(e.kind(), std::io::ErrorKind::Other); - assert_eq!(e.get_ref().unwrap().to_string(), "expected failure"); - }, - _ => panic!("unexpected result"), - } - fs::remove_dir_all(path).unwrap(); - } - - // Test failure to create the temporary file in the persistence process. - // We induce this failure by having the temp file already exist and be a - // directory. - #[test] - fn test_tmp_file_creation_failure() { - let test_writeable = TestWriteable {}; - let filename = "test_tmp_file_creation_failure_filename".to_string(); - let path = PathBuf::from("test_tmp_file_creation_failure_dir"); - - // Create the tmp file and make it a directory. - let tmp_path = get_full_filepath(path.clone(), format!("{}.tmp", filename.clone())); - fs::create_dir_all(tmp_path).unwrap(); - match write_to_file(path.clone(), filename, &test_writeable) { - Err(e) => { - #[cfg(not(target_os = "windows"))] - assert_eq!(e.raw_os_error(), Some(libc::EISDIR)); - #[cfg(target_os = "windows")] - assert_eq!(e.kind(), io::ErrorKind::PermissionDenied); - }, - _ => panic!("Unexpected error message"), - } - fs::remove_dir_all(path).unwrap(); - } -} diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 4fed0635b9..0f1e3e4085 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -178,38 +178,6 @@ macro_rules! ok_or_continue_after_sleep { }; } -#[cfg(not(target_arch = "wasm32"))] -macro_rules! ok_or_retry_after_sleep { - ($e:expr, $delay: ident) => { - loop { - match $e { - Ok(res) => break res, - Err(e) => { - error!("error {:?}", e); - Timer::sleep($delay).await; - continue; - }, - } - } - }; -} - -#[cfg(not(target_arch = "wasm32"))] -macro_rules! ok_or_retry_after_sleep_sync { - ($e:expr, $delay: ident) => { - loop { - match $e { - Ok(res) => break res, - Err(e) => { - error!("error {:?}", e); - std::thread::sleep(core::time::Duration::from_secs($delay)); - continue; - }, - } - } - }; -} - pub mod coin_balance; #[doc(hidden)] #[cfg(test)] diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index c1841ef111..ec7d1830f0 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -31,6 +31,7 @@ mod bchd_pb; pub mod qtum; pub mod rpc_clients; pub mod slp; +pub mod spv; pub mod utxo_block_header_storage; pub mod utxo_builder; pub mod utxo_common; @@ -69,8 +70,9 @@ use primitives::hash::{H256, H264}; use rpc::v1::types::{Bytes as BytesJson, Transaction as RpcTransaction, H256 as H256Json}; use script::{Builder, Script, SignatureVersion, TransactionInputSigner}; use serde_json::{self as json, Value as Json}; -use serialization::{serialize, serialize_with_flags, SERIALIZE_TRANSACTION_WITNESS}; +use serialization::{serialize, serialize_with_flags, Error as SerError, SERIALIZE_TRANSACTION_WITNESS}; use spv_validation::helpers_validation::SPVError; +use spv_validation::storage::BlockHeaderStorageError; use std::array::TryFromSliceError; use std::collections::{HashMap, HashSet}; use std::convert::TryInto; @@ -90,6 +92,7 @@ use utxo_signer::{TxProvider, TxProviderError, UtxoSignTxError, UtxoSignTxResult use self::rpc_clients::{electrum_script_hash, ElectrumClient, ElectrumRpcRequest, EstimateFeeMethod, EstimateFeeMode, NativeClient, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; +use self::utxo_block_header_storage::BlockHeaderVerificationParams; use super::{big_decimal_from_sat_unsigned, BalanceError, BalanceFut, BalanceResult, CoinBalance, CoinsContext, DerivationMethod, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, KmdRewardsDetails, MarketCoinOps, MmCoin, NumConversError, NumConversResult, PrivKeyActivationPolicy, PrivKeyNotAllowed, PrivKeyPolicy, @@ -101,9 +104,7 @@ use crate::coin_balance::{EnableCoinScanPolicy, HDAddressBalanceScanner}; use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError}; use crate::hd_wallet_storage::{HDAccountStorageItem, HDWalletCoinStorage, HDWalletStorageError, HDWalletStorageResult}; use crate::utxo::tx_cache::UtxoVerboseCacheShared; -use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; use crate::TransactionErr; -use utxo_block_header_storage::BlockHeaderStorage; pub mod tx_cache; #[cfg(target_arch = "wasm32")] @@ -530,7 +531,6 @@ pub struct UtxoCoinFields { pub history_sync_state: Mutex, /// The cache of verbose transactions. pub tx_cache: UtxoVerboseCacheShared, - pub block_headers_storage: Option, /// The cache of recently send transactions used to track the spent UTXOs and replace them with new outputs /// The daemon needs some time to update the listunspent list for address which makes it return already spent UTXOs /// This cache helps to prevent UTXO reuse in such cases @@ -566,27 +566,51 @@ impl From for WithdrawError { fn from(e: UnsupportedAddr) -> Self { WithdrawError::InvalidAddress(e.to_string()) } } +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub enum GetTxError { + Rpc(UtxoRpcError), + TxDeserialization(SerError), +} + +impl From for GetTxError { + fn from(err: UtxoRpcError) -> GetTxError { GetTxError::Rpc(err) } +} + +impl From for GetTxError { + fn from(err: SerError) -> GetTxError { GetTxError::TxDeserialization(err) } +} + #[derive(Debug)] pub enum GetTxHeightError { - HeightNotFound, + HeightNotFound(String), } impl From for SPVError { fn from(e: GetTxHeightError) -> Self { match e { - GetTxHeightError::HeightNotFound => SPVError::InvalidHeight, + GetTxHeightError::HeightNotFound(e) => SPVError::InvalidHeight(e), } } } -#[derive(Debug)] +impl From for GetTxHeightError { + fn from(e: UtxoRpcError) -> Self { GetTxHeightError::HeightNotFound(e.to_string()) } +} + +#[derive(Debug, Display)] pub enum GetBlockHeaderError { + #[display(fmt = "Block header storage error: {}", _0)] StorageError(BlockHeaderStorageError), + #[display(fmt = "RPC error: {}", _0)] RpcError(JsonRpcError), + #[display(fmt = "Serialization error: {}", _0)] SerializationError(serialization::Error), + #[display(fmt = "Invalid response: {}", _0)] InvalidResponse(String), + #[display(fmt = "Error validating headers: {}", _0)] SPVError(SPVError), - NativeNotSupported(String), + #[display(fmt = "Internal error: {}", _0)] Internal(String), } @@ -604,10 +628,6 @@ impl From for GetBlockHeaderError { } } -impl From for GetBlockHeaderError { - fn from(e: SPVError) -> Self { GetBlockHeaderError::SPVError(e) } -} - impl From for GetBlockHeaderError { fn from(err: serialization::Error) -> Self { GetBlockHeaderError::SerializationError(err) } } @@ -616,6 +636,10 @@ impl From for GetBlockHeaderError { fn from(err: BlockHeaderStorageError) -> Self { GetBlockHeaderError::StorageError(err) } } +impl From for SPVError { + fn from(e: GetBlockHeaderError) -> Self { SPVError::UnableToGetHeader(e.to_string()) } +} + impl UtxoCoinFields { pub fn transaction_preimage(&self) -> TransactionInputSigner { let lock_time = if self.conf.ticker == "KMD" { @@ -1184,14 +1208,6 @@ pub struct UtxoMergeParams { pub max_merge_at_once: usize, } -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct UtxoBlockHeaderVerificationParams { - pub difficulty_check: bool, - pub constant_difficulty: bool, - pub blocks_limit_to_check: NonZeroU64, - pub check_every: f64, -} - #[derive(Clone, Debug, Deserialize, Serialize)] pub struct UtxoActivationParams { pub mode: UtxoRpcMode, @@ -1232,7 +1248,12 @@ impl UtxoActivationParams { Some("electrum") => { let servers = json::from_value(req["servers"].clone()).map_to_mm(UtxoFromLegacyReqErr::InvalidElectrumServers)?; - UtxoRpcMode::Electrum { servers } + let block_header_params = json::from_value(req["block_header_params"].clone()) + .map_to_mm(UtxoFromLegacyReqErr::InvalidBlockHeaderVerificationParams)?; + UtxoRpcMode::Electrum { + servers, + block_header_params, + } }, _ => return MmError::err(UtxoFromLegacyReqErr::UnexpectedMethod), }; @@ -1274,7 +1295,10 @@ impl UtxoActivationParams { #[serde(tag = "rpc", content = "rpc_data")] pub enum UtxoRpcMode { Native, - Electrum { servers: Vec }, + Electrum { + servers: Vec, + block_header_params: Option, + }, } #[derive(Debug)] diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index cd9ba229af..70adae0cc2 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -12,8 +12,9 @@ use crate::rpc_command::init_create_account::{self, CreateNewAccountParams, Init use crate::rpc_command::init_scan_for_new_addresses::{self, InitScanAddressesRpcOps, ScanAddressesParams, ScanAddressesResponse}; use crate::rpc_command::init_withdraw::{InitWithdrawCoin, WithdrawTaskHandle}; -use crate::utxo::utxo_builder::{MergeUtxoArcOps, UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, - UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; +use crate::utxo::utxo_builder::{BlockHeaderUtxoArcOps, MergeUtxoArcOps, UtxoCoinBuildError, UtxoCoinBuilder, + UtxoCoinBuilderCommonOps, UtxoFieldsWithHardwareWalletBuilder, + UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::{eth, CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, DelegationError, DelegationFut, GetWithdrawSenderAddress, NegotiateSwapContractAddrErr, PrivKeyBuildPolicy, SearchForSwapTxSpendInput, SignatureResult, StakingInfosFut, SwapOps, TradePreimageValue, TransactionFut, UnexpectedDerivationMethod, @@ -216,17 +217,29 @@ impl<'a> UtxoCoinBuilder for QtumCoinBuilder<'a> { async fn build(self) -> MmResult { let utxo = self.build_utxo_fields().await?; + let rpc_client = utxo.rpc_client.clone(); let utxo_arc = UtxoArc::new(utxo); let utxo_weak = utxo_arc.downgrade(); let result_coin = QtumCoin::from(utxo_arc); - self.spawn_merge_utxo_loop_if_required(utxo_weak, QtumCoin::from); + if let Some(abort_handler) = self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), QtumCoin::from) { + self.ctx.abort_handlers.lock().unwrap().push(abort_handler); + } + + if let Some(abort_handler) = + self.spawn_block_header_utxo_loop_if_required(utxo_weak, &rpc_client, QtumCoin::from) + { + self.ctx.abort_handlers.lock().unwrap().push(abort_handler); + } + Ok(result_coin) } } impl<'a> MergeUtxoArcOps for QtumCoinBuilder<'a> {} +impl<'a> BlockHeaderUtxoArcOps for QtumCoinBuilder<'a> {} + impl<'a> QtumCoinBuilder<'a> { pub fn new( ctx: &'a MmArc, diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index fe9d5f819c..d2ac37ef24 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -1,7 +1,8 @@ #![cfg_attr(target_arch = "wasm32", allow(unused_macros))] #![cfg_attr(target_arch = "wasm32", allow(dead_code))] -use crate::utxo::{output_script, sat_from_big_decimal}; +use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; +use crate::utxo::{output_script, sat_from_big_decimal, GetBlockHeaderError, GetTxError, GetTxHeightError}; use crate::{big_decimal_from_sat_unsigned, NumConversError, RpcTransportEventHandler, RpcTransportEventHandlerShared}; use async_trait::async_trait; use chain::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, OutPoint, Transaction as UtxoTx}; @@ -34,8 +35,11 @@ use serde_json::{self as json, Value as Json}; use serialization::{deserialize, serialize, serialize_with_flags, CoinVariant, CompactInteger, Reader, SERIALIZE_TRANSACTION_WITNESS}; use sha2::{Digest, Sha256}; +use spv_validation::helpers_validation::{validate_headers, SPVError}; +use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::convert::TryInto; use std::fmt; use std::io; use std::net::{SocketAddr, ToSocketAddrs}; @@ -63,6 +67,8 @@ cfg_native! { use webpki_roots::TLS_SERVER_ROOTS; } +pub const NO_TX_ERROR_CODE: &str = "'code': -5"; + pub type AddressesByLabelResult = HashMap; pub type JsonRpcPendingRequestsShared = Arc>; pub type JsonRpcPendingRequests = HashMap>; @@ -338,6 +344,28 @@ pub trait UtxoRpcClientOps: fmt::Debug + Send + Sync + 'static { /// Returns block time in seconds since epoch (Jan 1 1970 GMT). async fn get_block_timestamp(&self, height: u64) -> Result>; + + /// Returns verbose transaction by the given `txid` if it's on-chain or None if it's not. + async fn get_tx_if_onchain(&self, tx_hash: &H256Json) -> Result, MmError> { + match self + .get_transaction_bytes(tx_hash) + .compat() + .await + .map_err(|e| e.into_inner()) + { + Ok(bytes) => Ok(Some(deserialize(bytes.as_slice())?)), + Err(err) => { + if let UtxoRpcError::ResponseParseError(ref json_err) = err { + if let JsonRpcErrorType::Response(_, json) = &json_err.error { + if json["message"].as_str().unwrap_or_default().contains(NO_TX_ERROR_CODE) { + return Ok(None); + } + } + } + Err(err.into()) + }, + } + } } #[derive(Clone, Deserialize, Debug)] @@ -1538,6 +1566,7 @@ pub struct ElectrumClientImpl { protocol_version: OrdRange, get_balance_concurrent_map: ConcurrentRequestMap, list_unspent_concurrent_map: ConcurrentRequestMap>, + block_headers_storage: Option, } async fn electrum_request_multi( @@ -1678,6 +1707,9 @@ impl ElectrumClientImpl { /// Get available protocol versions. pub fn protocol_version(&self) -> &OrdRange { &self.protocol_version } + + /// Get block headers storage. + pub fn block_headers_storage(&self) -> &Option { &self.block_headers_storage } } #[derive(Clone, Debug)] @@ -1883,6 +1915,93 @@ impl ElectrumClient { pub fn blockchain_transaction_get_merkle(&self, txid: H256Json, height: u64) -> RpcRes { rpc_func!(self, "blockchain.transaction.get_merkle", txid, height) } + + async fn get_tx_height(&self, tx: &UtxoTx) -> Result> { + for output in tx.outputs.clone() { + let script_pubkey_str = hex::encode(electrum_script_hash(&output.script_pubkey)); + if let Ok(history) = self.scripthash_get_history(script_pubkey_str.as_str()).compat().await { + if let Some(item) = history + .into_iter() + .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) + { + return Ok(item.height as u64); + } + } + } + MmError::err(GetTxHeightError::HeightNotFound( + "Couldn't find height through electrum!".into(), + )) + } + + async fn tx_height_from_storage_or_rpc(&self, tx: &UtxoTx) -> Result> { + if let Some(storage) = &self.block_headers_storage { + let ticker = self.coin_name(); + let tx_hash = tx.hash().reversed(); + let blockhash = self.get_verbose_transaction(&tx_hash.into()).compat().await?.blockhash; + if let Ok(Some(height)) = storage.get_block_height_by_hash(ticker, blockhash.into()).await { + if let Ok(height) = height.try_into() { + return Ok(height); + } + } + } + + self.get_tx_height(tx).await + } + + async fn valid_block_header_from_storage(&self, height: u64) -> Result> { + let storage = match &self.block_headers_storage { + Some(storage) => storage, + None => { + return MmError::err(GetBlockHeaderError::StorageError(BlockHeaderStorageError::Internal( + "block_headers_storage is not initialized".to_owned(), + ))) + }, + }; + let ticker = self.coin_name(); + match storage.get_block_header(ticker, height).await? { + None => { + let bytes = self.blockchain_block_header(height).compat().await?; + let header: BlockHeader = deserialize(bytes.0.as_slice())?; + let params = &storage.params; + let blocks_limit = params.blocks_limit_to_check; + let (headers_registry, headers) = self.retrieve_last_headers(blocks_limit, height).compat().await?; + match validate_headers(headers, params.difficulty_check, params.constant_difficulty) { + Ok(_) => { + storage.add_block_headers_to_storage(ticker, headers_registry).await?; + Ok(header) + }, + Err(err) => MmError::err(GetBlockHeaderError::SPVError(err)), + } + }, + Some(header) => Ok(header), + } + } + + async fn block_header_from_storage_or_rpc(&self, height: u64) -> Result> { + match &self.block_headers_storage { + Some(_) => self.valid_block_header_from_storage(height).await, + None => Ok(deserialize( + self.blockchain_block_header(height).compat().await?.as_slice(), + )?), + } + } + + pub async fn get_merkle_and_header( + &self, + tx: &UtxoTx, + ) -> Result<(TxMerkleBranch, BlockHeader, u64), MmError> { + let height = self.tx_height_from_storage_or_rpc(tx).await?; + + let merkle_branch = self + .blockchain_transaction_get_merkle(tx.hash().reversed().into(), height) + .compat() + .await + .map_to_mm(|e| SPVError::UnableToGetMerkle(e.to_string()))?; + + let header = self.block_header_from_storage_or_rpc(height).await?; + + Ok((merkle_branch, header, height)) + } } // if mockable is placed before async_trait there is `munmap_chunk(): invalid pointer` error on async fn mocking attempt @@ -2114,7 +2233,11 @@ impl UtxoRpcClientOps for ElectrumClient { #[cfg_attr(test, mockable)] impl ElectrumClientImpl { - pub fn new(coin_ticker: String, event_handlers: Vec) -> ElectrumClientImpl { + pub fn new( + coin_ticker: String, + event_handlers: Vec, + block_headers_storage: Option, + ) -> ElectrumClientImpl { let protocol_version = OrdRange::new(1.2, 1.4).unwrap(); ElectrumClientImpl { coin_ticker, @@ -2124,6 +2247,7 @@ impl ElectrumClientImpl { protocol_version, get_balance_concurrent_map: ConcurrentRequestMap::new(), list_unspent_concurrent_map: ConcurrentRequestMap::new(), + block_headers_storage, } } @@ -2132,10 +2256,11 @@ impl ElectrumClientImpl { coin_ticker: String, event_handlers: Vec, protocol_version: OrdRange, + block_headers_storage: Option, ) -> ElectrumClientImpl { ElectrumClientImpl { protocol_version, - ..ElectrumClientImpl::new(coin_ticker, event_handlers) + ..ElectrumClientImpl::new(coin_ticker, event_handlers, block_headers_storage) } } } diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index 9ed15d4843..6af6a0ea13 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -41,7 +41,7 @@ use rpc::v1::types::{Bytes as BytesJson, ToTxHash, H256 as H256Json}; use script::bytes::Bytes; use script::{Builder as ScriptBuilder, Opcode, Script, TransactionInputSigner}; use serde_json::Value as Json; -use serialization::{deserialize, serialize, Deserializable, Error, Reader}; +use serialization::{deserialize, serialize, Deserializable, Error as SerError, Reader}; use serialization_derive::Deserializable; use std::convert::TryInto; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; @@ -102,7 +102,7 @@ struct SlpTxPreimage { enum ValidateHtlcError { TxLackOfOutputs, #[display(fmt = "TxParseError: {:?}", _0)] - TxParseError(Error), + TxParseError(SerError), #[display(fmt = "OpReturnParseError: {:?}", _0)] OpReturnParseError(ParseSlpScriptError), InvalidSlpDetails, @@ -186,7 +186,7 @@ impl From for SpendP2SHError { pub enum SpendHtlcError { TxLackOfOutputs, #[display(fmt = "DeserializationErr: {:?}", _0)] - DeserializationErr(Error), + DeserializationErr(SerError), #[display(fmt = "PubkeyParseError: {:?}", _0)] PubkeyParseErr(keys::Error), InvalidSlpDetails, @@ -206,8 +206,8 @@ impl From for SpendHtlcError { fn from(err: NumConversError) -> SpendHtlcError { SpendHtlcError::NumConversionErr(err) } } -impl From for SpendHtlcError { - fn from(err: Error) -> SpendHtlcError { SpendHtlcError::DeserializationErr(err) } +impl From for SpendHtlcError { + fn from(err: SerError) -> SpendHtlcError { SpendHtlcError::DeserializationErr(err) } } impl From for SpendHtlcError { @@ -815,7 +815,7 @@ impl SlpTransaction { } impl Deserializable for SlpTransaction { - fn deserialize(reader: &mut Reader) -> Result + fn deserialize(reader: &mut Reader) -> Result where Self: Sized, T: std::io::Read, @@ -831,7 +831,7 @@ impl Deserializable for SlpTransaction { } else { let mut url = vec![0; maybe_push_op_code as usize]; reader.read_slice(&mut url)?; - String::from_utf8(url).map_err(|e| Error::Custom(e.to_string()))? + String::from_utf8(url).map_err(|e| SerError::Custom(e.to_string()))? }; let maybe_push_op_code: u8 = reader.read()?; @@ -852,7 +852,7 @@ impl Deserializable for SlpTransaction { }; let bytes: Vec = reader.read_list()?; if bytes.len() != 8 { - return Err(Error::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); + return Err(SerError::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); } let initial_token_mint_quantity = u64::from_be_bytes(bytes.try_into().expect("length is 8 bytes")); @@ -869,7 +869,10 @@ impl Deserializable for SlpTransaction { SLP_MINT => { let maybe_id: Vec = reader.read_list()?; if maybe_id.len() != 32 { - return Err(Error::Custom(format!("Unexpected token id length {}", maybe_id.len()))); + return Err(SerError::Custom(format!( + "Unexpected token id length {}", + maybe_id.len() + ))); } let maybe_push_op_code: u8 = reader.read()?; @@ -882,7 +885,7 @@ impl Deserializable for SlpTransaction { let bytes: Vec = reader.read_list()?; if bytes.len() != 8 { - return Err(Error::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); + return Err(SerError::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); } let additional_token_quantity = u64::from_be_bytes(bytes.try_into().expect("length is 8 bytes")); @@ -895,7 +898,10 @@ impl Deserializable for SlpTransaction { SLP_SEND => { let maybe_id: Vec = reader.read_list()?; if maybe_id.len() != 32 { - return Err(Error::Custom(format!("Unexpected token id length {}", maybe_id.len()))); + return Err(SerError::Custom(format!( + "Unexpected token id length {}", + maybe_id.len() + ))); } let token_id = H256::from(maybe_id.as_slice()); @@ -903,21 +909,21 @@ impl Deserializable for SlpTransaction { while !reader.is_finished() { let bytes: Vec = reader.read_list()?; if bytes.len() != 8 { - return Err(Error::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); + return Err(SerError::Custom(format!("Expected 8 bytes, got {}", bytes.len()))); } let amount = u64::from_be_bytes(bytes.try_into().expect("length is 8 bytes")); amounts.push(amount) } if amounts.len() > 19 { - return Err(Error::Custom(format!( + return Err(SerError::Custom(format!( "Expected at most 19 token amounts, got {}", amounts.len() ))); } Ok(SlpTransaction::Send { token_id, amounts }) }, - _ => Err(Error::Custom(format!( + _ => Err(SerError::Custom(format!( "Unsupported transaction type {}", transaction_type ))), @@ -940,11 +946,11 @@ pub enum ParseSlpScriptError { #[display(fmt = "UnexpectedTokenType: {:?}", _0)] UnexpectedTokenType(Vec), #[display(fmt = "DeserializeFailed: {:?}", _0)] - DeserializeFailed(Error), + DeserializeFailed(SerError), } -impl From for ParseSlpScriptError { - fn from(err: Error) -> ParseSlpScriptError { ParseSlpScriptError::DeserializeFailed(err) } +impl From for ParseSlpScriptError { + fn from(err: SerError) -> ParseSlpScriptError { ParseSlpScriptError::DeserializeFailed(err) } } pub fn parse_slp_script(script: &[u8]) -> Result> { diff --git a/mm2src/coins/utxo/spv.rs b/mm2src/coins/utxo/spv.rs new file mode 100644 index 0000000000..6d85d0afbe --- /dev/null +++ b/mm2src/coins/utxo/spv.rs @@ -0,0 +1,91 @@ +use crate::utxo::rpc_clients::ElectrumClient; +use async_trait::async_trait; +use chain::{BlockHeader, RawBlockHeader, Transaction as UtxoTx}; +use common::executor::Timer; +use common::log::error; +use common::now_ms; +use keys::hash::H256; +use mm2_err_handle::prelude::*; +use serialization::serialize_list; +use spv_validation::helpers_validation::SPVError; +use spv_validation::spv_proof::{SPVProof, TRY_SPV_PROOF_INTERVAL}; + +pub struct ConfirmedTransactionInfo { + pub tx: UtxoTx, + pub header: BlockHeader, + pub index: u64, + pub height: u64, +} + +#[async_trait] +pub trait SimplePaymentVerification { + async fn validate_spv_proof( + &self, + tx: &UtxoTx, + try_spv_proof_until: u64, + ) -> Result>; +} + +#[async_trait] +impl SimplePaymentVerification for ElectrumClient { + async fn validate_spv_proof( + &self, + tx: &UtxoTx, + try_spv_proof_until: u64, + ) -> Result> { + if tx.outputs.is_empty() { + return MmError::err(SPVError::InvalidVout); + } + + let (merkle_branch, header, height) = loop { + if now_ms() / 1000 > try_spv_proof_until { + error!( + "Waited too long until {} for transaction {:?} to validate spv proof", + try_spv_proof_until, + tx.hash().reversed(), + ); + return MmError::err(SPVError::Timeout); + } + + match self.get_merkle_and_header(tx).await { + Ok(res) => break res, + Err(e) => { + error!( + "Failed spv proof validation for transaction {} with error: {:?}, retrying in {} seconds.", + tx.hash().reversed(), + e, + TRY_SPV_PROOF_INTERVAL, + ); + + Timer::sleep(TRY_SPV_PROOF_INTERVAL as f64).await; + }, + } + }; + + let raw_header = RawBlockHeader::new(header.raw().take())?; + let intermediate_nodes: Vec = merkle_branch + .merkle + .into_iter() + .map(|hash| hash.reversed().into()) + .collect(); + + let proof = SPVProof { + tx_id: tx.hash(), + vin: serialize_list(&tx.inputs).take(), + vout: serialize_list(&tx.outputs).take(), + index: merkle_branch.pos as u64, + confirming_header: header.clone(), + raw_header, + intermediate_nodes, + }; + + proof.validate().map_err(MmError::new)?; + + Ok(ConfirmedTransactionInfo { + tx: tx.clone(), + header, + index: proof.index, + height, + }) + } +} diff --git a/mm2src/coins/utxo/utxo_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage.rs index 925a0c80c3..32f53dd89f 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage.rs @@ -1,40 +1,29 @@ -use crate::utxo::rpc_clients::ElectrumBlockHeader; #[cfg(target_arch = "wasm32")] use crate::utxo::utxo_indexedb_block_header_storage::IndexedDBBlockHeadersStorage; #[cfg(not(target_arch = "wasm32"))] use crate::utxo::utxo_sql_block_header_storage::SqliteBlockHeadersStorage; -use crate::utxo::UtxoBlockHeaderVerificationParams; use async_trait::async_trait; use chain::BlockHeader; -use derive_more::Display; use mm2_core::mm_ctx::MmArc; -use mm2_err_handle::prelude::*; +use primitives::hash::H256; +use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use std::collections::HashMap; use std::fmt::{Debug, Formatter}; - -#[derive(Debug, Display)] -pub enum BlockHeaderStorageError { - #[display(fmt = "Can't add to the storage for {} - reason: {}", ticker, reason)] - AddToStorageError { ticker: String, reason: String }, - #[display(fmt = "Can't get from the storage for {} - reason: {}", ticker, reason)] - GetFromStorageError { ticker: String, reason: String }, - #[display( - fmt = "Can't retrieve the table from the storage for {} - reason: {}", - ticker, - reason - )] - CantRetrieveTableError { ticker: String, reason: String }, - #[display(fmt = "Can't query from the storage - query: {} - reason: {}", query, reason)] - QueryError { query: String, reason: String }, - #[display(fmt = "Can't init from the storage - ticker: {} - reason: {}", ticker, reason)] - InitializationError { ticker: String, reason: String }, - #[display(fmt = "Can't decode/deserialize from storage for {} - reason: {}", ticker, reason)] - DecodeError { ticker: String, reason: String }, +use std::num::NonZeroU64; + +/// SPV headers verification parameters +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct BlockHeaderVerificationParams { + pub difficulty_check: bool, + pub constant_difficulty: bool, + // This should to be equal to or greater than the number of blocks needed before the chain is safe from reorganization (e.g. 6 blocks for BTC) + pub blocks_limit_to_check: NonZeroU64, + pub check_every: f64, } pub struct BlockHeaderStorage { pub inner: Box, - pub params: UtxoBlockHeaderVerificationParams, + pub params: BlockHeaderVerificationParams, } impl Debug for BlockHeaderStorage { @@ -42,63 +31,29 @@ impl Debug for BlockHeaderStorage { } pub trait InitBlockHeaderStorageOps: Send + Sync + 'static { - fn new_from_ctx(ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option + fn new_from_ctx( + ctx: MmArc, + params: BlockHeaderVerificationParams, + ) -> Result where Self: Sized; } -#[async_trait] -pub trait BlockHeaderStorageOps: Send + Sync + 'static { - /// Initializes collection/tables in storage for a specified coin - async fn init(&self, for_coin: &str) -> Result<(), MmError>; - - async fn is_initialized_for(&self, for_coin: &str) -> Result>; - - // Adds multiple block headers to the selected coin's header storage - // Should store it as `TICKER_HEIGHT=hex_string` - // use this function for headers that comes from `blockchain_headers_subscribe` - async fn add_electrum_block_headers_to_storage( - &self, - for_coin: &str, - headers: Vec, - ) -> Result<(), MmError>; - - // Adds multiple block headers to the selected coin's header storage - // Should store it as `TICKER_HEIGHT=hex_string` - // use this function for headers that comes from `blockchain_block_headers` - async fn add_block_headers_to_storage( - &self, - for_coin: &str, - headers: HashMap, - ) -> Result<(), MmError>; - - /// Gets the block header by height from the selected coin's storage as BlockHeader - async fn get_block_header( - &self, - for_coin: &str, - height: u64, - ) -> Result, MmError>; - - /// Gets the block header by height from the selected coin's storage as hex - async fn get_block_header_raw( - &self, - for_coin: &str, - height: u64, - ) -> Result, MmError>; -} - impl InitBlockHeaderStorageOps for BlockHeaderStorage { #[cfg(not(target_arch = "wasm32"))] - fn new_from_ctx(ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { - ctx.sqlite_connection.as_option().map(|connection| BlockHeaderStorage { - inner: Box::new(SqliteBlockHeadersStorage(connection.clone())), + fn new_from_ctx(ctx: MmArc, params: BlockHeaderVerificationParams) -> Result { + let sqlite_connection = ctx.sqlite_connection.ok_or(BlockHeaderStorageError::Internal( + "sqlite_connection is not initialized".to_owned(), + ))?; + Ok(BlockHeaderStorage { + inner: Box::new(SqliteBlockHeadersStorage(sqlite_connection.clone())), params, }) } #[cfg(target_arch = "wasm32")] - fn new_from_ctx(_ctx: MmArc, params: UtxoBlockHeaderVerificationParams) -> Option { - Some(BlockHeaderStorage { + fn new_from_ctx(_ctx: MmArc, params: BlockHeaderVerificationParams) -> Result { + Ok(BlockHeaderStorage { inner: Box::new(IndexedDBBlockHeadersStorage {}), params, }) @@ -107,29 +62,17 @@ impl InitBlockHeaderStorageOps for BlockHeaderStorage { #[async_trait] impl BlockHeaderStorageOps for BlockHeaderStorage { - async fn init(&self, for_coin: &str) -> Result<(), MmError> { - self.inner.init(for_coin).await - } + async fn init(&self, for_coin: &str) -> Result<(), BlockHeaderStorageError> { self.inner.init(for_coin).await } - async fn is_initialized_for(&self, for_coin: &str) -> Result> { + async fn is_initialized_for(&self, for_coin: &str) -> Result { self.inner.is_initialized_for(for_coin).await } - async fn add_electrum_block_headers_to_storage( - &self, - for_coin: &str, - headers: Vec, - ) -> Result<(), MmError> { - self.inner - .add_electrum_block_headers_to_storage(for_coin, headers) - .await - } - async fn add_block_headers_to_storage( &self, for_coin: &str, headers: HashMap, - ) -> Result<(), MmError> { + ) -> Result<(), BlockHeaderStorageError> { self.inner.add_block_headers_to_storage(for_coin, headers).await } @@ -137,7 +80,7 @@ impl BlockHeaderStorageOps for BlockHeaderStorage { &self, for_coin: &str, height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { self.inner.get_block_header(for_coin, height).await } @@ -145,7 +88,22 @@ impl BlockHeaderStorageOps for BlockHeaderStorage { &self, for_coin: &str, height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { self.inner.get_block_header_raw(for_coin, height).await } + + async fn get_last_block_header_with_non_max_bits( + &self, + for_coin: &str, + ) -> Result, BlockHeaderStorageError> { + self.inner.get_last_block_header_with_non_max_bits(for_coin).await + } + + async fn get_block_height_by_hash( + &self, + for_coin: &str, + hash: H256, + ) -> Result, BlockHeaderStorageError> { + self.inner.get_block_height_by_hash(for_coin, hash).await + } } diff --git a/mm2src/coins/utxo/utxo_builder/mod.rs b/mm2src/coins/utxo/utxo_builder/mod.rs index 9c1cf135d0..cd48444513 100644 --- a/mm2src/coins/utxo/utxo_builder/mod.rs +++ b/mm2src/coins/utxo/utxo_builder/mod.rs @@ -2,7 +2,7 @@ mod utxo_arc_builder; mod utxo_coin_builder; mod utxo_conf_builder; -pub use utxo_arc_builder::{MergeUtxoArcOps, UtxoArcBuilder}; +pub use utxo_arc_builder::{BlockHeaderUtxoArcOps, MergeUtxoArcOps, UtxoArcBuilder}; pub use utxo_coin_builder::{UtxoCoinBuildError, UtxoCoinBuildResult, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index d4964b12c4..2a90bc036f 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -1,4 +1,4 @@ -use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; +use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{block_header_utxo_loop, merge_utxo_loop}; @@ -84,16 +84,19 @@ where async fn build(self) -> MmResult { let utxo = self.build_utxo_fields().await?; + let rpc_client = utxo.rpc_client.clone(); let utxo_arc = UtxoArc::new(utxo); let utxo_weak = utxo_arc.downgrade(); let result_coin = (self.constructor)(utxo_arc); - self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()); - if let Some(abort_handler) = self.spawn_block_header_utxo_loop_if_required( - utxo_weak, - &result_coin.as_ref().block_headers_storage, - self.constructor.clone(), - ) { + if let Some(abort_handler) = self.spawn_merge_utxo_loop_if_required(utxo_weak.clone(), self.constructor.clone()) + { + self.ctx.abort_handlers.lock().unwrap().push(abort_handler); + } + + if let Some(abort_handler) = + self.spawn_block_header_utxo_loop_if_required(utxo_weak, &rpc_client, self.constructor.clone()) + { self.ctx.abort_handlers.lock().unwrap().push(abort_handler); } Ok(result_coin) @@ -115,21 +118,28 @@ where } pub trait MergeUtxoArcOps: UtxoCoinBuilderCommonOps { - fn spawn_merge_utxo_loop_if_required(&self, weak: UtxoWeak, constructor: F) + fn spawn_merge_utxo_loop_if_required(&self, weak: UtxoWeak, constructor: F) -> Option where F: Fn(UtxoArc) -> T + Send + Sync + 'static, { if let Some(ref merge_params) = self.activation_params().utxo_merge_params { - let fut = merge_utxo_loop( + let (fut, abort_handle) = abortable(merge_utxo_loop( weak, merge_params.merge_at, merge_params.check_every, merge_params.max_merge_at_once, constructor, - ); - info!("Starting UTXO merge loop for coin {}", self.ticker()); - spawn(fut); + )); + let ticker = self.ticker().to_owned(); + info!("Starting UTXO merge loop for coin {}", ticker); + spawn(async move { + if let Err(e) = fut.await { + info!("spawn_merge_utxo_loop_if_required stopped for {}, reason {}", ticker, e); + } + }); + return Some(abort_handle); } + None } } @@ -137,26 +147,28 @@ pub trait BlockHeaderUtxoArcOps: UtxoCoinBuilderCommonOps { fn spawn_block_header_utxo_loop_if_required( &self, weak: UtxoWeak, - maybe_storage: &Option, + rpc_client: &UtxoRpcClientEnum, constructor: F, ) -> Option where F: Fn(UtxoArc) -> T + Send + Sync + 'static, T: UtxoCommonOps, { - if maybe_storage.is_some() { - let ticker = self.ticker().to_owned(); - let (fut, abort_handle) = abortable(block_header_utxo_loop(weak, constructor)); - info!("Starting UTXO block header loop for coin {}", ticker); - spawn(async move { - if let Err(e) = fut.await { - info!( - "spawn_block_header_utxo_loop_if_required stopped for {}, reason {}", - ticker, e - ); - } - }); - return Some(abort_handle); + if let UtxoRpcClientEnum::Electrum(electrum) = rpc_client { + if electrum.block_headers_storage().is_some() { + let ticker = self.ticker().to_owned(); + let (fut, abort_handle) = abortable(block_header_utxo_loop(weak, constructor)); + info!("Starting UTXO block header loop for coin {}", ticker); + spawn(async move { + if let Err(e) = fut.await { + info!( + "spawn_block_header_utxo_loop_if_required stopped for {}, reason {}", + ticker, e + ); + } + }); + return Some(abort_handle); + } } None } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index dac910db93..faced1c8f1 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -3,7 +3,8 @@ use crate::hd_wallet_storage::{HDWalletCoinStorage, HDWalletStorageError}; use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, ElectrumRpcRequest, EstimateFeeMethod, UtxoRpcClientEnum}; use crate::utxo::tx_cache::{UtxoVerboseCacheOps, UtxoVerboseCacheShared}; -use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, InitBlockHeaderStorageOps}; +use crate::utxo::utxo_block_header_storage::{BlockHeaderStorage, BlockHeaderVerificationParams, + InitBlockHeaderStorageOps}; use crate::utxo::utxo_builder::utxo_conf_builder::{UtxoConfBuilder, UtxoConfError, UtxoConfResult}; use crate::utxo::{output_script, utxo_common, ElectrumBuilderArgs, ElectrumProtoVerifier, RecentlySpentOutPoints, TxFee, UtxoCoinConf, UtxoCoinFields, UtxoHDAccount, UtxoHDWallet, UtxoRpcMode, DEFAULT_GAP_LIMIT, @@ -163,7 +164,6 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); let tx_cache = self.tx_cache(); - let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { conf, @@ -174,7 +174,6 @@ pub trait UtxoFieldsWithIguanaPrivKeyBuilder: UtxoCoinBuilderCommonOps { derivation_method, history_sync_state: Mutex::new(initial_history_state), tx_cache, - block_headers_storage, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_fee, tx_hash_algo, @@ -226,7 +225,6 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { let tx_hash_algo = self.tx_hash_algo(); let check_utxo_maturity = self.check_utxo_maturity(); let tx_cache = self.tx_cache(); - let block_headers_storage = self.block_headers_storage()?; let coin = UtxoCoinFields { conf, @@ -236,7 +234,6 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { priv_key_policy: PrivKeyPolicy::Trezor, derivation_method: DerivationMethod::HDWallet(hd_wallet), history_sync_state: Mutex::new(initial_history_state), - block_headers_storage, tx_cache, recently_spent_outpoints, tx_fee, @@ -289,15 +286,6 @@ pub trait UtxoCoinBuilderCommonOps { fn ticker(&self) -> &str; - fn block_headers_storage(&self) -> UtxoCoinBuildResult> { - let params: Option<_> = json::from_value(self.conf()["block_header_params"].clone()) - .map_to_mm(|e| UtxoConfError::InvalidBlockHeaderParams(e.to_string()))?; - match params { - None => Ok(None), - Some(params) => Ok(BlockHeaderStorage::new_from_ctx(self.ctx().clone(), params)), - } - } - fn address_format(&self) -> UtxoCoinBuildResult { let format_from_req = self.activation_params().address_format.clone(); let format_from_conf = json::from_value::>(self.conf()["address_format"].clone()) @@ -406,8 +394,13 @@ pub trait UtxoCoinBuilderCommonOps { Ok(UtxoRpcClientEnum::Native(native)) } }, - UtxoRpcMode::Electrum { servers } => { - let electrum = self.electrum_client(ElectrumBuilderArgs::default(), servers).await?; + UtxoRpcMode::Electrum { + servers, + block_header_params, + } => { + let electrum = self + .electrum_client(ElectrumBuilderArgs::default(), servers, block_header_params) + .await?; Ok(UtxoRpcClientEnum::Electrum(electrum)) }, } @@ -417,6 +410,7 @@ pub trait UtxoCoinBuilderCommonOps { &self, args: ElectrumBuilderArgs, mut servers: Vec, + block_header_params: Option, ) -> UtxoCoinBuildResult { let (on_connect_tx, on_connect_rx) = mpsc::unbounded(); let ticker = self.ticker().to_owned(); @@ -432,9 +426,17 @@ pub trait UtxoCoinBuilderCommonOps { event_handlers.push(ElectrumProtoVerifier { on_connect_tx }.into_shared()); } + let block_headers_storage = match block_header_params { + Some(params) => Some( + BlockHeaderStorage::new_from_ctx(self.ctx().clone(), params) + .map_to_mm(|e| UtxoCoinBuildError::Internal(e.to_string()))?, + ), + None => None, + }; + let mut rng = small_rng(); servers.as_mut_slice().shuffle(&mut rng); - let client = ElectrumClientImpl::new(ticker, event_handlers); + let client = ElectrumClientImpl::new(ticker, event_handlers, block_headers_storage); for server in servers.iter() { match client.add_server(server).await { Ok(_) => (), diff --git a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs index 3cc89efe7b..db1512b38d 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_conf_builder.rs @@ -40,7 +40,6 @@ pub enum UtxoConfError { InvalidConsensusBranchId(String), InvalidVersionGroupId(String), InvalidAddressFormat(String), - InvalidBlockHeaderParams(String), InvalidDecimals(String), } diff --git a/mm2src/coins/utxo/utxo_common.rs b/mm2src/coins/utxo/utxo_common.rs index a67c9136c4..583ed7f73b 100644 --- a/mm2src/coins/utxo/utxo_common.rs +++ b/mm2src/coins/utxo/utxo_common.rs @@ -1,4 +1,3 @@ -use super::rpc_clients::TxMerkleBranch; use super::*; use crate::coin_balance::{AddressBalanceStatus, HDAddressBalance, HDWalletBalanceOps}; use crate::hd_pubkey::{ExtractExtendedPubkey, HDExtractPubkeyError, HDXPubExtractor}; @@ -8,6 +7,7 @@ use crate::hd_wallet_storage::{HDWalletCoinWithStorageOps, HDWalletStorageResult use crate::rpc_command::init_withdraw::WithdrawTaskHandle; use crate::utxo::rpc_clients::{electrum_script_hash, BlockHashOrHeight, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcClientOps, UtxoRpcResult}; +use crate::utxo::spv::SimplePaymentVerification; use crate::utxo::tx_cache::TxCacheResult; use crate::utxo::utxo_withdraw::{InitUtxoWithdraw, StandardUtxoWithdraw, UtxoWithdraw}; use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSenderAddress, HDAddressId, @@ -18,7 +18,7 @@ use crate::{CanRefundHtlc, CoinBalance, CoinWithDerivationMethod, GetWithdrawSen use bitcrypto::dhash256; pub use bitcrypto::{dhash160, sha256, ChecksumType}; use chain::constants::SEQUENCE_FINAL; -use chain::{BlockHeader, OutPoint, RawBlockHeader, TransactionOutput}; +use chain::{OutPoint, TransactionOutput}; use common::executor::Timer; use common::jsonrpc_client::JsonRpcErrorType; use common::log::{debug, error, info, warn}; @@ -40,16 +40,14 @@ use rpc::v1::types::{Bytes as BytesJson, ToTxHash, TransactionInputEnum, H256 as use script::{Builder, Opcode, Script, ScriptAddress, TransactionInputSigner, UnsignedTransactionInput}; use secp256k1::{PublicKey, Signature}; use serde_json::{self as json}; -use serialization::{deserialize, serialize, serialize_list, serialize_with_flags, CoinVariant, CompactInteger, - Serializable, Stream, SERIALIZE_TRANSACTION_WITNESS}; +use serialization::{deserialize, serialize, serialize_with_flags, CoinVariant, CompactInteger, Serializable, Stream, + SERIALIZE_TRANSACTION_WITNESS}; use spv_validation::helpers_validation::validate_headers; -use spv_validation::helpers_validation::SPVError; -use spv_validation::spv_proof::{SPVProof, TRY_SPV_PROOF_INTERVAL}; +use spv_validation::storage::BlockHeaderStorageOps; use std::cmp::Ordering; use std::collections::hash_map::{Entry, HashMap}; use std::str::FromStr; use std::sync::atomic::Ordering as AtomicOrdering; -use utxo_block_header_storage::BlockHeaderStorageOps; use utxo_signer::with_key_pair::p2sh_spend; use utxo_signer::UtxoSignerOps; @@ -59,7 +57,6 @@ pub const DEFAULT_FEE_VOUT: usize = 0; pub const DEFAULT_SWAP_TX_SPEND_SIZE: u64 = 305; pub const DEFAULT_SWAP_VOUT: usize = 0; const MIN_BTC_TRADING_VOL: &str = "0.00777"; -pub const NO_TX_ERROR_CODE: &str = "'code': -5"; macro_rules! true_or { ($cond: expr, $etype: expr) => { @@ -3078,130 +3075,6 @@ pub fn address_from_pubkey( } } -pub async fn validate_spv_proof( - coin: T, - tx: UtxoTx, - try_spv_proof_until: u64, -) -> Result<(), MmError> { - let client = match &coin.as_ref().rpc_client { - UtxoRpcClientEnum::Native(_) => return Ok(()), - UtxoRpcClientEnum::Electrum(electrum_client) => electrum_client, - }; - if tx.outputs.is_empty() { - return MmError::err(SPVError::InvalidVout); - } - - let (merkle_branch, block_header) = spv_proof_retry_pool(&coin, client, &tx, try_spv_proof_until).await?; - let raw_header = RawBlockHeader::new(block_header.raw().take())?; - let intermediate_nodes: Vec = merkle_branch - .merkle - .into_iter() - .map(|hash| hash.reversed().into()) - .collect(); - - let proof = SPVProof { - tx_id: tx.hash(), - vin: serialize_list(&tx.inputs).take(), - vout: serialize_list(&tx.outputs).take(), - index: merkle_branch.pos as u64, - confirming_header: block_header, - raw_header, - intermediate_nodes, - }; - - proof.validate().map_err(MmError::new) -} - -async fn spv_proof_retry_pool( - coin: &T, - client: &ElectrumClient, - tx: &UtxoTx, - try_spv_proof_until: u64, -) -> Result<(TxMerkleBranch, BlockHeader), MmError> { - let mut height: Option = None; - let mut merkle_branch: Option = None; - - loop { - if now_ms() / 1000 > try_spv_proof_until { - error!( - "Waited too long until {} for transaction {:?} to validate spv proof", - try_spv_proof_until, - tx.hash(), - ); - return Err(SPVError::Timeout.into()); - } - - if height.is_none() { - match get_tx_height(tx, client).await { - Ok(h) => height = Some(h), - Err(e) => { - debug!("`get_tx_height` returned an error {:?}", e); - error!("{:?} for tx {:?}", SPVError::InvalidHeight, tx); - }, - } - } - - if height.is_some() && merkle_branch.is_none() { - match client - .blockchain_transaction_get_merkle(tx.hash().reversed().into(), height.unwrap()) - .compat() - .await - { - Ok(m) => merkle_branch = Some(m), - Err(e) => { - debug!("`blockchain_transaction_get_merkle` returned an error {:?}", e); - error!( - "{:?} by tx: {:?}, height: {}", - SPVError::UnableToGetMerkle, - H256Json::from(tx.hash().reversed()), - height.unwrap() - ); - }, - } - } - - if height.is_some() && merkle_branch.is_some() { - match block_header_from_storage_or_rpc(&coin, height.unwrap(), &coin.as_ref().block_headers_storage, client) - .await - { - Ok(block_header) => { - return Ok((merkle_branch.unwrap(), block_header)); - }, - Err(e) => { - debug!("`block_header_from_storage_or_rpc` returned an error {:?}", e); - error!( - "{:?}, Received header likely not compatible with header format in mm2", - SPVError::UnableToGetHeader - ); - }, - } - } - - error!( - "Failed spv proof validation for transaction {:?}, retrying in {} seconds.", - tx.hash(), - TRY_SPV_PROOF_INTERVAL, - ); - - Timer::sleep(TRY_SPV_PROOF_INTERVAL as f64).await; - } -} - -pub async fn get_tx_height(tx: &UtxoTx, client: &ElectrumClient) -> Result> { - for output in tx.outputs.clone() { - let script_pubkey_str = hex::encode(electrum_script_hash(&output.script_pubkey)); - if let Ok(history) = client.scripthash_get_history(script_pubkey_str.as_str()).compat().await { - if let Some(item) = history - .into_iter() - .find(|item| item.tx_hash.reversed() == H256Json(*tx.hash()) && item.height > 0) - { - return Ok(item.height as u64); - } - } - } - MmError::err(GetTxHeightError::HeightNotFound) -} - #[allow(clippy::too_many_arguments)] #[cfg_attr(test, mockable)] pub fn validate_payment( @@ -3268,16 +3141,16 @@ pub fn validate_payment( ); } - if !coin.as_ref().conf.enable_spv_proof { - return Ok(()); + if let UtxoRpcClientEnum::Electrum(client) = &coin.as_ref().rpc_client { + if coin.as_ref().conf.enable_spv_proof && confirmations != 0 { + client + .validate_spv_proof(&tx, try_spv_proof_until) + .await + .map_err(|e| format!("{:?}", e))?; + } } - return match confirmations { - 0 => Ok(()), - _ => validate_spv_proof(coin, tx, try_spv_proof_until) - .await - .map_err(|e| format!("{:?}", e)), - }; + return Ok(()); } }; Box::new(fut.boxed().compat()) @@ -3561,61 +3434,6 @@ fn increase_by_percent(num: u64, percent: f64) -> u64 { num + (percent.round() as u64) } -pub async fn valid_block_header_from_storage( - coin: &T, - height: u64, - storage: &BlockHeaderStorage, - client: &ElectrumClient, -) -> Result> -where - T: AsRef, -{ - match storage - .get_block_header(coin.as_ref().conf.ticker.as_str(), height) - .await? - { - None => { - let bytes = client.blockchain_block_header(height).compat().await?; - let header: BlockHeader = deserialize(bytes.0.as_slice())?; - let params = &storage.params; - let blocks_limit = params.blocks_limit_to_check; - let (headers_registry, headers) = client.retrieve_last_headers(blocks_limit, height).compat().await?; - match spv_validation::helpers_validation::validate_headers( - headers, - params.difficulty_check, - params.constant_difficulty, - ) { - Ok(_) => { - storage - .add_block_headers_to_storage(coin.as_ref().conf.ticker.as_str(), headers_registry) - .await?; - Ok(header) - }, - Err(err) => MmError::err(GetBlockHeaderError::SPVError(err)), - } - }, - Some(header) => Ok(header), - } -} - -#[inline] -pub async fn block_header_from_storage_or_rpc( - coin: &T, - height: u64, - storage: &Option, - client: &ElectrumClient, -) -> Result> -where - T: AsRef, -{ - match storage { - Some(ref storage) => valid_block_header_from_storage(&coin, height, storage, client).await, - None => Ok(deserialize( - client.blockchain_block_header(height).compat().await?.as_slice(), - )?), - } -} - pub async fn block_header_utxo_loop(weak: UtxoWeak, constructor: impl Fn(UtxoArc) -> T) { { let coin = match weak.upgrade() { @@ -3623,9 +3441,12 @@ pub async fn block_header_utxo_loop(weak: UtxoWeak, constructo None => return, }; let ticker = coin.as_ref().conf.ticker.as_str(); - let storage = match &coin.as_ref().block_headers_storage { - None => return, - Some(storage) => storage, + let storage = match &coin.as_ref().rpc_client { + UtxoRpcClientEnum::Native(_) => return, + UtxoRpcClientEnum::Electrum(e) => match e.block_headers_storage() { + None => return, + Some(storage) => storage, + }, }; match storage.is_initialized_for(ticker).await { Ok(true) => info!("Block Header Storage already initialized for {}", ticker), @@ -3644,8 +3465,12 @@ pub async fn block_header_utxo_loop(weak: UtxoWeak, constructo } while let Some(arc) = weak.upgrade() { let coin = constructor(arc); - let storage = match &coin.as_ref().block_headers_storage { - None => break, + let client = match &coin.as_ref().rpc_client { + UtxoRpcClientEnum::Native(_) => break, + UtxoRpcClientEnum::Electrum(client) => client, + }; + let storage = match client.block_headers_storage() { + None => return, Some(storage) => storage, }; let params = storage.params.clone(); @@ -3657,10 +3482,6 @@ pub async fn block_header_utxo_loop(weak: UtxoWeak, constructo ); let height = ok_or_continue_after_sleep!(coin.as_ref().rpc_client.get_block_count().compat().await, check_every); - let client = match &coin.as_ref().rpc_client { - UtxoRpcClientEnum::Native(_) => break, - UtxoRpcClientEnum::Electrum(client) => client, - }; let (block_registry, block_headers) = ok_or_continue_after_sleep!( client .retrieve_last_headers(blocks_limit_to_check, height) diff --git a/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs index d00d4f261f..e0e98658a4 100644 --- a/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_indexedb_block_header_storage.rs @@ -1,9 +1,7 @@ -use crate::utxo::rpc_clients::ElectrumBlockHeader; -use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError; -use crate::utxo::utxo_block_header_storage::BlockHeaderStorageOps; use async_trait::async_trait; use chain::BlockHeader; -use mm2_err_handle::prelude::*; +use primitives::hash::H256; +use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use std::collections::HashMap; #[derive(Debug)] @@ -11,23 +9,15 @@ pub struct IndexedDBBlockHeadersStorage {} #[async_trait] impl BlockHeaderStorageOps for IndexedDBBlockHeadersStorage { - async fn init(&self, _for_coin: &str) -> Result<(), MmError> { Ok(()) } + async fn init(&self, _for_coin: &str) -> Result<(), BlockHeaderStorageError> { Ok(()) } - async fn is_initialized_for(&self, _for_coin: &str) -> Result> { Ok(true) } - - async fn add_electrum_block_headers_to_storage( - &self, - _for_coin: &str, - _headers: Vec, - ) -> Result<(), MmError> { - Ok(()) - } + async fn is_initialized_for(&self, _for_coin: &str) -> Result { Ok(true) } async fn add_block_headers_to_storage( &self, _for_coin: &str, _headers: HashMap, - ) -> Result<(), MmError> { + ) -> Result<(), BlockHeaderStorageError> { Ok(()) } @@ -35,7 +25,7 @@ impl BlockHeaderStorageOps for IndexedDBBlockHeadersStorage { &self, _for_coin: &str, _height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { Ok(None) } @@ -43,7 +33,22 @@ impl BlockHeaderStorageOps for IndexedDBBlockHeadersStorage { &self, _for_coin: &str, _height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { + Ok(None) + } + + async fn get_last_block_header_with_non_max_bits( + &self, + _for_coin: &str, + ) -> Result, BlockHeaderStorageError> { + Ok(None) + } + + async fn get_block_height_by_hash( + &self, + for_coin: &str, + hash: H256, + ) -> Result, BlockHeaderStorageError> { Ok(None) } } diff --git a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs index d11b794e54..c73d0c40de 100644 --- a/mm2src/coins/utxo/utxo_sql_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_sql_block_header_storage.rs @@ -1,5 +1,3 @@ -use crate::utxo::rpc_clients::ElectrumBlockHeader; -use crate::utxo::utxo_block_header_storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use async_trait::async_trait; use chain::BlockHeader; use common::async_blocking; @@ -8,28 +6,32 @@ use db_common::{sqlite::rusqlite::Error as SqlError, sqlite::string_from_row, sqlite::validate_table_name, sqlite::CHECK_TABLE_EXISTS_SQL}; -use mm2_err_handle::prelude::*; -use serialization::deserialize; +use primitives::hash::H256; +use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; +use spv_validation::work::MAX_BITS_BTC; use std::collections::HashMap; +use std::convert::TryInto; use std::sync::{Arc, Mutex}; fn block_headers_cache_table(ticker: &str) -> String { ticker.to_owned() + "_block_headers_cache" } -fn get_table_name_and_validate(for_coin: &str) -> Result> { +fn get_table_name_and_validate(for_coin: &str) -> Result { let table_name = block_headers_cache_table(for_coin); validate_table_name(&table_name).map_err(|e| BlockHeaderStorageError::CantRetrieveTableError { - ticker: for_coin.to_string(), + coin: for_coin.to_string(), reason: e.to_string(), })?; Ok(table_name) } -fn create_block_header_cache_table_sql(for_coin: &str) -> Result> { +fn create_block_header_cache_table_sql(for_coin: &str) -> Result { let table_name = get_table_name_and_validate(for_coin)?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( block_height INTEGER NOT NULL UNIQUE, - hex TEXT NOT NULL + hex TEXT NOT NULL, + block_bits INTEGER NOT NULL, + block_hash VARCHAR(255) NOT NULL UNIQUE );", table_name ); @@ -37,23 +39,40 @@ fn create_block_header_cache_table_sql(for_coin: &str) -> Result Result> { +fn insert_block_header_in_cache_sql(for_coin: &str) -> Result { let table_name = get_table_name_and_validate(for_coin)?; // Always update the block headers with new values just in case a chain reorganization occurs. let sql = format!( - "INSERT OR REPLACE INTO {} (block_height, hex) VALUES (?1, ?2);", + "INSERT OR REPLACE INTO {} (block_height, hex, block_bits, block_hash) VALUES (?1, ?2, ?3, ?4);", table_name ); Ok(sql) } -fn get_block_header_by_height(for_coin: &str) -> Result> { +fn get_block_header_by_height(for_coin: &str) -> Result { let table_name = get_table_name_and_validate(for_coin)?; let sql = format!("SELECT hex FROM {} WHERE block_height=?1;", table_name); Ok(sql) } +fn get_last_block_header_with_non_max_bits_sql(for_coin: &str) -> Result { + let table_name = get_table_name_and_validate(for_coin)?; + let sql = format!( + "SELECT hex FROM {} WHERE block_bits<>{} ORDER BY block_height DESC LIMIT 1;", + table_name, MAX_BITS_BTC + ); + + Ok(sql) +} + +fn get_block_height_by_hash(for_coin: &str) -> Result { + let table_name = get_table_name_and_validate(for_coin)?; + let sql = format!("SELECT block_height FROM {} WHERE block_hash=?1;", table_name); + + Ok(sql) +} + #[derive(Clone, Debug)] pub struct SqliteBlockHeadersStorage(pub Arc>); @@ -62,89 +81,29 @@ fn query_single_row( query: &str, params: P, map_fn: F, -) -> Result, MmError> +) -> Result, BlockHeaderStorageError> where P: IntoIterator, P::Item: ToSql, F: FnOnce(&Row<'_>) -> Result, { - db_common::sqlite::query_single_row(conn, query, params, map_fn).map_err(|e| { - MmError::new(BlockHeaderStorageError::QueryError { - query: query.to_string(), - reason: e.to_string(), - }) + db_common::sqlite::query_single_row(conn, query, params, map_fn).map_err(|e| BlockHeaderStorageError::QueryError { + query: query.to_string(), + reason: e.to_string(), }) } -struct SqlBlockHeader { - block_height: String, - block_hex: String, -} - -impl From for SqlBlockHeader { - fn from(header: ElectrumBlockHeader) -> Self { - match header { - ElectrumBlockHeader::V12(h) => { - let block_hex = h.as_hex(); - let block_height = h.block_height.to_string(); - SqlBlockHeader { - block_height, - block_hex, - } - }, - ElectrumBlockHeader::V14(h) => { - let block_hex = format!("{:02x}", h.hex); - let block_height = h.height.to_string(); - SqlBlockHeader { - block_height, - block_hex, - } - }, - } - } -} -async fn common_headers_insert( - for_coin: &str, - storage: SqliteBlockHeadersStorage, - headers: Vec, -) -> Result<(), MmError> { - let for_coin = for_coin.to_owned(); - let mut conn = storage.0.lock().unwrap(); - let sql_transaction = conn - .transaction() - .map_err(|e| BlockHeaderStorageError::AddToStorageError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - for header in headers { - let block_cache_params = [&header.block_height, &header.block_hex]; - sql_transaction - .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) - .map_err(|e| BlockHeaderStorageError::AddToStorageError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - } - sql_transaction - .commit() - .map_err(|e| BlockHeaderStorageError::AddToStorageError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; - Ok(()) -} - #[async_trait] impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { - async fn init(&self, for_coin: &str) -> Result<(), MmError> { + async fn init(&self, for_coin: &str) -> Result<(), BlockHeaderStorageError> { let selfi = self.clone(); let sql_cache = create_block_header_cache_table_sql(for_coin)?; - let ticker = for_coin.to_owned(); + let coin = for_coin.to_owned(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); conn.execute(&sql_cache, NO_PARAMS).map(|_| ()).map_err(|e| { BlockHeaderStorageError::InitializationError { - ticker, + coin, reason: e.to_string(), } })?; @@ -153,7 +112,7 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { .await } - async fn is_initialized_for(&self, for_coin: &str) -> Result> { + async fn is_initialized_for(&self, for_coin: &str) -> Result { let block_headers_cache_table = get_table_name_and_validate(for_coin)?; let selfi = self.clone(); async_blocking(move || { @@ -169,45 +128,64 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { .await } - async fn add_electrum_block_headers_to_storage( - &self, - for_coin: &str, - headers: Vec, - ) -> Result<(), MmError> { - let headers_for_sql = headers.into_iter().map(Into::into).collect(); - common_headers_insert(for_coin, self.clone(), headers_for_sql).await - } - async fn add_block_headers_to_storage( &self, for_coin: &str, headers: HashMap, - ) -> Result<(), MmError> { - let headers_for_sql = headers - .into_iter() - .map(|(height, header)| SqlBlockHeader { - block_height: height.to_string(), - block_hex: hex::encode(header.raw()), - }) - .collect(); - common_headers_insert(for_coin, self.clone(), headers_for_sql).await + ) -> Result<(), BlockHeaderStorageError> { + let for_coin = for_coin.to_owned(); + let selfi = self.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn + .transaction() + .map_err(|e| BlockHeaderStorageError::AddToStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + + for (height, header) in headers { + let height = height as i64; + let hash = header.hash().reversed().to_string(); + let raw_header = hex::encode(header.raw()); + let bits: u32 = header.bits.into(); + let block_cache_params = [ + &height as &dyn ToSql, + &raw_header as &dyn ToSql, + &bits as &dyn ToSql, + &hash as &dyn ToSql, + ]; + sql_transaction + .execute(&insert_block_header_in_cache_sql(&for_coin)?, block_cache_params) + .map_err(|e| BlockHeaderStorageError::AddToStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + } + sql_transaction + .commit() + .map_err(|e| BlockHeaderStorageError::AddToStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + Ok(()) + }) + .await } async fn get_block_header( &self, for_coin: &str, height: u64, - ) -> Result, MmError> { + ) -> Result, BlockHeaderStorageError> { if let Some(header_raw) = self.get_block_header_raw(for_coin, height).await? { - let header_bytes = hex::decode(header_raw).map_err(|e| BlockHeaderStorageError::DecodeError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; let header: BlockHeader = - deserialize(header_bytes.as_slice()).map_err(|e| BlockHeaderStorageError::DecodeError { - ticker: for_coin.to_string(), - reason: e.to_string(), - })?; + header_raw + .try_into() + .map_err(|e: serialization::Error| BlockHeaderStorageError::DecodeError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; return Ok(Some(header)); } Ok(None) @@ -217,8 +195,8 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { &self, for_coin: &str, height: u64, - ) -> Result, MmError> { - let params = [height.to_string()]; + ) -> Result, BlockHeaderStorageError> { + let params = [height as i64]; let sql = get_block_header_by_height(for_coin)?; let selfi = self.clone(); @@ -227,11 +205,59 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage { query_single_row(&conn, &sql, params, string_from_row) }) .await - .map_err(|e| { - MmError::new(BlockHeaderStorageError::GetFromStorageError { - ticker: for_coin.to_string(), - reason: e.into_inner().to_string(), - }) + .map_err(|e| BlockHeaderStorageError::GetFromStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + }) + } + + async fn get_last_block_header_with_non_max_bits( + &self, + for_coin: &str, + ) -> Result, BlockHeaderStorageError> { + let sql = get_last_block_header_with_non_max_bits_sql(for_coin)?; + let selfi = self.clone(); + + let maybe_header_raw = async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, NO_PARAMS, string_from_row) + }) + .await + .map_err(|e| BlockHeaderStorageError::GetFromStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + + if let Some(header_raw) = maybe_header_raw { + let header: BlockHeader = + header_raw + .try_into() + .map_err(|e: serialization::Error| BlockHeaderStorageError::DecodeError { + coin: for_coin.to_string(), + reason: e.to_string(), + })?; + return Ok(Some(header)); + } + Ok(None) + } + + async fn get_block_height_by_hash( + &self, + for_coin: &str, + hash: H256, + ) -> Result, BlockHeaderStorageError> { + let params = [hash.to_string()]; + let sql = get_block_height_by_hash(for_coin)?; + let selfi = self.clone(); + + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, params, |row| row.get(0)) + }) + .await + .map_err(|e| BlockHeaderStorageError::GetFromStorageError { + coin: for_coin.to_string(), + reason: e.to_string(), }) } } @@ -254,7 +280,7 @@ impl SqliteBlockHeadersStorage { #[cfg(test)] mod sql_block_headers_storage_tests { use super::*; - use crate::utxo::rpc_clients::ElectrumBlockHeaderV14; + use chain::BlockHeaderBits; use common::block_on; use primitives::hash::H256; @@ -283,12 +309,10 @@ mod sql_block_headers_storage_tests { let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); assert!(initialized); - let block_header = ElectrumBlockHeaderV14 { - height: 520481, - hex: "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(), - }.into(); - let headers = vec![ElectrumBlockHeader::V14(block_header)]; - block_on(storage.add_electrum_block_headers_to_storage(for_coin, headers)).unwrap(); + let mut headers = HashMap::with_capacity(1); + let block_header: BlockHeader = "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(); + headers.insert(520481, block_header); + block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); assert!(!storage.is_table_empty(&table)); } @@ -302,12 +326,11 @@ mod sql_block_headers_storage_tests { let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); assert!(initialized); - let block_header = ElectrumBlockHeaderV14 { - height: 520481, - hex: "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(), - }.into(); - let headers = vec![ElectrumBlockHeader::V14(block_header)]; - block_on(storage.add_electrum_block_headers_to_storage(for_coin, headers)).unwrap(); + let mut headers = HashMap::with_capacity(1); + let block_header: BlockHeader = "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".into(); + headers.insert(520481, block_header); + + block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); assert!(!storage.is_table_empty(&table)); let hex = block_on(storage.get_block_header_raw(for_coin, 520481)) @@ -316,9 +339,48 @@ mod sql_block_headers_storage_tests { assert_eq!(hex, "0000002076d41d3e4b0bfd4c0d3b30aa69fdff3ed35d85829efd04000000000000000000b386498b583390959d9bac72346986e3015e83ac0b54bc7747a11a494ac35c94bb3ce65a53fb45177f7e311c".to_string()); let block_header = block_on(storage.get_block_header(for_coin, 520481)).unwrap().unwrap(); - assert_eq!( - block_header.hash(), - H256::from_reversed_str("0000000000000000002e31d0714a5ab23100945ff87ba2d856cd566a3c9344ec") - ) + let block_hash: H256 = "0000000000000000002e31d0714a5ab23100945ff87ba2d856cd566a3c9344ec".into(); + assert_eq!(block_header.hash(), block_hash.reversed()); + + let height = block_on(storage.get_block_height_by_hash(for_coin, block_hash)) + .unwrap() + .unwrap(); + assert_eq!(height, 520481); + } + + #[test] + fn test_get_last_block_header_with_non_max_bits() { + let for_coin = "get"; + let storage = SqliteBlockHeadersStorage::in_memory(); + let table = block_headers_cache_table(for_coin); + block_on(storage.init(for_coin)).unwrap(); + + let initialized = block_on(storage.is_initialized_for(for_coin)).unwrap(); + assert!(initialized); + + let mut headers = HashMap::with_capacity(2); + + // This block has max difficulty + // https://live.blockcypher.com/btc-testnet/block/00000000961a9d117feb57e516e17217207a849bf6cdfce529f31d9a96053530/ + let block_header: BlockHeader = "02000000ea01a61a2d7420a1b23875e40eb5eb4ca18b378902c8e6384514ad0000000000c0c5a1ae80582b3fe319d8543307fa67befc2a734b8eddb84b1780dfdf11fa2b20e71353ffff001d00805fe0".into(); + headers.insert(201595, block_header); + + // https://live.blockcypher.com/btc-testnet/block/0000000000ad144538e6c80289378ba14cebb50ee47538b2a120742d1aa601ea/ + let expected_block_header: BlockHeader = "02000000cbed7fd98f1f06e85c47e13ff956533642056be45e7e6b532d4d768f00000000f2680982f333fcc9afa7f9a5e2a84dc54b7fe10605cd187362980b3aa882e9683be21353ab80011c813e1fc0".into(); + headers.insert(201594, expected_block_header.clone()); + + // This block has max difficulty + // https://live.blockcypher.com/btc-testnet/block/0000000000ad144538e6c80289378ba14cebb50ee47538b2a120742d1aa601ea/ + let block_header: BlockHeader = "020000001f38c8e30b30af912fbd4c3e781506713cfb43e73dff6250348e060000000000afa8f3eede276ccb4c4ee649ad9823fc181632f262848ca330733e7e7e541beb9be51353ffff001d00a63037".into(); + headers.insert(201593, block_header); + + block_on(storage.add_block_headers_to_storage(for_coin, headers)).unwrap(); + assert!(!storage.is_table_empty(&table)); + + let actual_block_header = block_on(storage.get_last_block_header_with_non_max_bits(for_coin)) + .unwrap() + .unwrap(); + assert_ne!(actual_block_header.bits, BlockHeaderBits::Compact(MAX_BITS_BTC.into())); + assert_eq!(actual_block_header, expected_block_header); } } diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 00f74b03ce..56dc7a9015 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -10,6 +10,7 @@ use crate::utxo::rpc_clients::{BlockHashOrHeight, ElectrumBalance, ElectrumClien GetAddressInfoRes, ListSinceBlockRes, ListTransactionsItem, NativeClient, NativeClientImpl, NativeUnspent, NetworkInfo, UtxoRpcClientOps, ValidateAddressRes, VerboseBlock}; +use crate::utxo::spv::SimplePaymentVerification; use crate::utxo::tx_cache::dummy_tx_cache::DummyVerboseCache; use crate::utxo::tx_cache::UtxoVerboseCacheOps; use crate::utxo::utxo_builder::{UtxoArcBuilder, UtxoCoinBuilderCommonOps}; @@ -69,7 +70,7 @@ pub fn electrum_client_for_test(servers: &[&str]) -> ElectrumClient { }; let servers = servers.into_iter().map(|s| json::from_value(s).unwrap()).collect(); - block_on(builder.electrum_client(args, servers)).unwrap() + block_on(builder.electrum_client(args, servers, None)).unwrap() } /// Returned client won't work by default, requires some mocks to be usable @@ -165,7 +166,6 @@ fn utxo_coin_fields_for_test( derivation_method, history_sync_state: Mutex::new(HistorySyncState::NotEnabled), tx_cache: DummyVerboseCache::default().into_shared(), - block_headers_storage: None, recently_spent_outpoints: AsyncMutex::new(RecentlySpentOutPoints::new(my_script_pubkey)), tx_hash_algo: TxHashAlgo::DSHA256, check_utxo_maturity: false, @@ -469,7 +469,7 @@ fn test_wait_for_payment_spend_timeout_electrum() { MockResult::Return(Box::new(futures01::future::ok(None))) }); - let client = ElectrumClientImpl::new(TEST_COIN_NAME.into(), Default::default()); + let client = ElectrumClientImpl::new(TEST_COIN_NAME.into(), Default::default(), None); let client = UtxoRpcClientEnum::Electrum(ElectrumClient(Arc::new(client))); let coin = utxo_coin_for_test(client, None, false); let transaction = hex::decode("01000000000102fff7f7881a8099afa6940d42d1e7f6362bec38171ea3edf433541db4e4ad969f00000000494830450221008b9d1dc26ba6a9cb62127b02742fa9d754cd3bebf337f7a55d114c8e5cdd30be022040529b194ba3f9281a99f2b1c0a19c0489bc22ede944ccf4ecbab4cc618ef3ed01eeffffffef51e1b804cc89d182d279655c3aa89e815b1b309fe287d9b2b55d57b90ec68a0100000000ffffffff02202cb206000000001976a9148280b37df378db99f66f85c95a783a76ac7a6d5988ac9093510d000000001976a9143bde42dbee7e4dbe6a21b2d50ce2f0167faa815988ac000247304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee0121025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee635711000000") @@ -956,18 +956,13 @@ fn test_utxo_lock() { #[test] fn test_spv_proof() { let client = electrum_client_for_test(RICK_ELECTRUM_ADDRS); - let coin = utxo_coin_for_test( - client.into(), - Some("spice describe gravity federal blast come thank unfair canal monkey style afraid"), - false, - ); // https://rick.explorer.dexstats.info/tx/78ea7839f6d1b0dafda2ba7e34c1d8218676a58bd1b33f03a5f76391f61b72b0 let tx_str = "0400008085202f8902bf17bf7d1daace52e08f732a6b8771743ca4b1cb765a187e72fd091a0aabfd52000000006a47304402203eaaa3c4da101240f80f9c5e9de716a22b1ec6d66080de6a0cca32011cd77223022040d9082b6242d6acf9a1a8e658779e1c655d708379862f235e8ba7b8ca4e69c6012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffffff023ca13c0e9e085dd13f481f193e8a3e8fd609020936e98b5587342d994f4d020000006b483045022100c0ba56adb8de923975052312467347d83238bd8d480ce66e8b709a7997373994022048507bcac921fdb2302fa5224ce86e41b7efc1a2e20ae63aa738dfa99b7be826012102031d4256c4bc9f99ac88bf3dba21773132281f65f9bf23a59928bce08961e2f3ffffffff0300e1f5050000000017a9141ee6d4c38a3c078eab87ad1a5e4b00f21259b10d870000000000000000166a1400000000000000000000000000000000000000001b94d736000000001976a91405aab5342166f8594baf17a7d9bef5d56744332788ac2d08e35e000000000000000000000000000000"; let tx: UtxoTx = tx_str.into(); - let res = block_on(utxo_common::validate_spv_proof(coin.clone(), tx, now_ms() / 1000 + 30)); - res.unwrap() + let res = block_on(client.validate_spv_proof(&tx, now_ms() / 1000 + 30)); + res.unwrap(); } #[test] @@ -1473,11 +1468,12 @@ fn test_network_info_negative_time_offset() { #[test] fn test_unavailable_electrum_proto_version() { - ElectrumClientImpl::new.mock_safe(|coin_ticker, event_handlers| { + ElectrumClientImpl::new.mock_safe(|coin_ticker, event_handlers, _| { MockResult::Return(ElectrumClientImpl::with_protocol_version( coin_ticker, event_handlers, OrdRange::new(1.8, 1.9).unwrap(), + None, )) }); diff --git a/mm2src/coins/utxo/utxo_wasm_tests.rs b/mm2src/coins/utxo/utxo_wasm_tests.rs index 4eaaf01d79..20c057c01b 100644 --- a/mm2src/coins/utxo/utxo_wasm_tests.rs +++ b/mm2src/coins/utxo/utxo_wasm_tests.rs @@ -11,7 +11,7 @@ wasm_bindgen_test_configure!(run_in_browser); const TEST_COIN_NAME: &'static str = "RICK"; pub async fn electrum_client_for_test(servers: &[&str]) -> ElectrumClient { - let client = ElectrumClientImpl::new(TEST_COIN_NAME.into(), Default::default()); + let client = ElectrumClientImpl::new(TEST_COIN_NAME.into(), Default::default(), None); for server in servers { client .add_server(&ElectrumRpcRequest { diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 7d859fca8f..0d287952a9 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -534,6 +534,8 @@ impl<'a> ZCoinBuilder<'a> { ZcoinRpcMode::Native => UtxoRpcMode::Native, ZcoinRpcMode::Light { electrum_servers, .. } => UtxoRpcMode::Electrum { servers: electrum_servers.clone(), + // TODO: Implement spv validation for zcoin + block_header_params: None, }, }; let utxo_params = UtxoActivationParams { diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 6a752bde42..404f229405 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,5 +1,5 @@ use super::{z_coin_errors::*, ZcoinConsensusParams}; -use crate::utxo::utxo_common; +use crate::utxo::rpc_clients; use common::executor::Timer; use common::log::{debug, error, info}; use common::{async_blocking, spawn_abortable, AbortOnDropHandle}; @@ -367,7 +367,7 @@ impl SaplingSyncLoopHandle { Ok(_) => break, Err(e) => { error!("Error on getting tx {}", tx_id); - if e.message().contains(utxo_common::NO_TX_ERROR_CODE) { + if e.message().contains(rpc_clients::NO_TX_ERROR_CODE) { if attempts >= 3 { self.watch_for_tx = None; return; diff --git a/mm2src/db_common/src/sqlite.rs b/mm2src/db_common/src/sqlite.rs index 8cb351f1db..0b18dfa54d 100644 --- a/mm2src/db_common/src/sqlite.rs +++ b/mm2src/db_common/src/sqlite.rs @@ -31,7 +31,7 @@ pub(crate) type OwnedSqlParam = Value; pub(crate) type OwnedSqlParams = Vec; type SqlNamedParam<'a> = (&'a str, &'a dyn ToSql); -type SqlNamedParams<'a> = Vec>; +pub type SqlNamedParams<'a> = Vec>; type OwnedSqlNamedParam = (&'static str, Value); pub type OwnedSqlNamedParams = Vec; diff --git a/mm2src/mm2_bitcoin/chain/src/block_header.rs b/mm2src/mm2_bitcoin/chain/src/block_header.rs index a3e92f6132..4de005f0b5 100644 --- a/mm2src/mm2_bitcoin/chain/src/block_header.rs +++ b/mm2src/mm2_bitcoin/chain/src/block_header.rs @@ -1,10 +1,13 @@ use compact::Compact; use crypto::dhash256; +use ext_bitcoin::blockdata::block::BlockHeader as ExtBlockHeader; +use ext_bitcoin::hash_types::{BlockHash as ExtBlockHash, TxMerkleNode as ExtTxMerkleNode}; use hash::H256; use hex::FromHex; use primitives::bytes::Bytes; use primitives::U256; use ser::{deserialize, serialize, Deserializable, Reader, Serializable, Stream}; +use std::convert::TryFrom; use std::io; use transaction::{deserialize_tx, TxType}; use {OutPoint, Transaction}; @@ -39,6 +42,24 @@ impl Serializable for BlockHeaderBits { } } +impl From for u32 { + fn from(bits: BlockHeaderBits) -> Self { + match bits { + BlockHeaderBits::Compact(c) => c.into(), + BlockHeaderBits::U32(n) => n, + } + } +} + +impl From for Compact { + fn from(bits: BlockHeaderBits) -> Self { + match bits { + BlockHeaderBits::Compact(c) => c, + BlockHeaderBits::U32(n) => Compact::new(n), + } + } +} + const AUX_POW_VERSION_DOGE: u32 = 6422788; const AUX_POW_VERSION_SYS: u32 = 537919744; const MTP_POW_VERSION: u32 = 0x20001000u32; @@ -303,8 +324,39 @@ impl From<&'static str> for BlockHeader { fn from(s: &'static str) -> Self { deserialize(&s.from_hex::>().unwrap() as &[u8]).unwrap() } } +impl TryFrom for BlockHeader { + type Error = ser::Error; + fn try_from(s: String) -> Result { + deserialize( + &s.from_hex::>() + .map_err(|e| Self::Error::Custom(e.to_string()))? as &[u8], + ) + } +} + +impl From for ExtBlockHeader { + fn from(header: BlockHeader) -> Self { + let prev_blockhash = ExtBlockHash::from_hash(header.previous_header_hash.to_sha256d()); + let merkle_root = ExtTxMerkleNode::from_hash(header.merkle_root_hash.to_sha256d()); + // note: H256 nonce is not supported for bitcoin, we will just set nonce to 0 in this case since this will never happen + let nonce = match header.nonce { + BlockHeaderNonce::U32(n) => n, + _ => 0, + }; + ExtBlockHeader { + version: header.version as i32, + prev_blockhash, + merkle_root, + time: header.time, + bits: header.bits.into(), + nonce, + } + } +} + #[cfg(test)] mod tests { + use super::ExtBlockHeader; use block_header::{BlockHeader, BlockHeaderBits, BlockHeaderNonce, AUX_POW_VERSION_DOGE, AUX_POW_VERSION_SYS, KAWPOW_VERSION, MTP_POW_VERSION, PROG_POW_SWITCH_TIME, QTUM_BLOCK_HEADER_VERSION}; use hex::FromHex; @@ -2305,4 +2357,15 @@ mod tests { let serialized = serialize_list(&headers); assert_eq!(serialized.take(), headers_bytes); } + + #[test] + fn test_from_blockheader_to_ext_blockheader() { + // https://live.blockcypher.com/btc/block/00000000000000000020cf2bdc6563fb25c424af588d5fb7223461e72715e4a9/ + let header: BlockHeader = "0200000066720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab610000000000000000007829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc548118751eb505ac1910018de19b302".into(); + let ext_header = ExtBlockHeader::from(header.clone()); + assert_eq!( + header.hash().reversed().to_string(), + ext_header.block_hash().to_string() + ); + } } diff --git a/mm2src/mm2_bitcoin/chain/src/raw_block.rs b/mm2src/mm2_bitcoin/chain/src/raw_block.rs index 075d15ad6a..c50e4c1e31 100644 --- a/mm2src/mm2_bitcoin/chain/src/raw_block.rs +++ b/mm2src/mm2_bitcoin/chain/src/raw_block.rs @@ -8,7 +8,7 @@ pub const MIN_RAW_HEADER_SIZE: usize = 80_usize; /// Hex-encoded block #[derive(Default, PartialEq, Clone, Eq, Hash)] -pub struct RawBlockHeader(Bytes); +pub struct RawBlockHeader(pub Bytes); #[derive(Debug, PartialEq, Eq, Clone)] pub enum RawHeaderError { diff --git a/mm2src/mm2_bitcoin/chain/src/transaction.rs b/mm2src/mm2_bitcoin/chain/src/transaction.rs index 37d84e09db..e2585275e2 100644 --- a/mm2src/mm2_bitcoin/chain/src/transaction.rs +++ b/mm2src/mm2_bitcoin/chain/src/transaction.rs @@ -4,13 +4,12 @@ use bytes::Bytes; use constants::{LOCKTIME_THRESHOLD, SEQUENCE_FINAL}; use crypto::{dhash256, sha256}; -use ext_bitcoin::blockdata::transaction::Transaction as ExtTransaction; -use ext_bitcoin::consensus::encode::{deserialize as deserialize_ext, Error as EncodeError}; +use ext_bitcoin::blockdata::transaction::{OutPoint as ExtOutpoint, Transaction as ExtTransaction, TxIn, TxOut}; +use ext_bitcoin::hash_types::Txid; use hash::{CipherText, EncCipherText, OutCipherText, ZkProof, ZkProofSapling, H256, H512, H64}; use hex::FromHex; use ser::{deserialize, serialize, serialize_with_flags, SERIALIZE_TRANSACTION_WITNESS}; use ser::{CompactInteger, Deserializable, Error, Reader, Serializable, Stream}; -use std::convert::TryFrom; use std::io; use std::io::Read; @@ -38,6 +37,15 @@ impl OutPoint { pub fn is_null(&self) -> bool { self.hash.is_zero() && self.index == u32::MAX } } +impl From for ExtOutpoint { + fn from(outpoint: OutPoint) -> Self { + ExtOutpoint { + txid: Txid::from_hash(outpoint.hash.to_sha256d()), + vout: outpoint.index, + } + } +} + #[derive(Debug, PartialEq, Default, Clone)] pub struct TransactionInput { pub previous_output: OutPoint, @@ -61,6 +69,17 @@ impl TransactionInput { pub fn has_witness(&self) -> bool { !self.script_witness.is_empty() } } +impl From for TxIn { + fn from(txin: TransactionInput) -> Self { + TxIn { + previous_output: txin.previous_output.into(), + script_sig: txin.script_sig.take().into(), + sequence: txin.sequence, + witness: txin.script_witness.into_iter().map(|s| s.take()).collect(), + } + } +} + #[derive(Debug, PartialEq, Clone, Serializable, Deserializable)] pub struct TransactionOutput { pub value: u64, @@ -76,6 +95,15 @@ impl Default for TransactionOutput { } } +impl From for TxOut { + fn from(txout: TransactionOutput) -> Self { + TxOut { + value: txout.value, + script_pubkey: txout.script_pubkey.take().into(), + } + } +} + #[derive(Debug, PartialEq, Clone, Serializable, Deserializable)] pub struct ShieldedSpend { pub cv: H256, @@ -198,16 +226,14 @@ impl From<&'static str> for Transaction { fn from(s: &'static str) -> Self { deserialize(&s.from_hex::>().unwrap() as &[u8]).unwrap() } } -impl TryFrom for ExtTransaction { - type Error = EncodeError; - - fn try_from(tx: Transaction) -> Result { - let tx_hex = if tx.has_witness() { - serialize_with_flags(&tx, SERIALIZE_TRANSACTION_WITNESS) - } else { - serialize(&tx) - }; - deserialize_ext(&tx_hex.take()) +impl From for ExtTransaction { + fn from(tx: Transaction) -> Self { + ExtTransaction { + version: tx.version, + lock_time: tx.lock_time, + input: tx.inputs.into_iter().map(|i| i.into()).collect(), + output: tx.outputs.into_iter().map(|o| o.into()).collect(), + } } } @@ -527,7 +553,6 @@ mod tests { use hash::{H256, H512}; use hex::ToHex; use ser::{deserialize, serialize, serialize_with_flags, Serializable, SERIALIZE_TRANSACTION_WITNESS}; - use std::convert::TryFrom; use TxHashAlgo; // real transaction from block 80000 @@ -995,10 +1020,10 @@ mod tests { } #[test] - fn test_try_from_tx_to_ext_tx() { + fn test_from_tx_to_ext_tx() { // https://live.blockcypher.com/btc-testnet/tx/2be90e03abb4d5328bf7e9467ca9c571aef575837b55f1253119b87e85ccb94f/ let tx: Transaction = "010000000001016546e6d844ad0142c8049a839e8deae16c17f0a6587e36e75ff2181ed7020a800100000000ffffffff0247070800000000002200200bbfbd271853ec0a775e5455d4bb19d32818e9b5bda50655ac183fb15c9aa01625910300000000001600149a85cc05e9a722575feb770a217c73fd6145cf0102473044022002eac5d11f3800131985c14a3d1bc03dfe5e694f5731bde39b0d2b183eb7d3d702201d62e7ff2dd433260bf7a8223db400d539a2c4eccd27a5aa24d83f5ad9e9e1750121031ac6d25833a5961e2a8822b2e8b0ac1fd55d90cbbbb18a780552cbd66fc02bb35c099c61".into(); - let ext_tx = ExtTransaction::try_from(tx.clone()).unwrap(); + let ext_tx = ExtTransaction::from(tx.clone()); assert_eq!(tx.hash().reversed().to_string(), ext_tx.txid().to_string()); } } diff --git a/mm2src/mm2_bitcoin/primitives/Cargo.toml b/mm2src/mm2_bitcoin/primitives/Cargo.toml index ef679deeb8..764f02d1fa 100644 --- a/mm2src/mm2_bitcoin/primitives/Cargo.toml +++ b/mm2src/mm2_bitcoin/primitives/Cargo.toml @@ -5,5 +5,6 @@ authors = ["debris "] [dependencies] rustc-hex = "2" +bitcoin_hashes = "0.10.0" byteorder = "1.0" -uint = "0.9.1" +uint = "0.9.3" diff --git a/mm2src/mm2_bitcoin/primitives/src/hash.rs b/mm2src/mm2_bitcoin/primitives/src/hash.rs index 785653a980..e7c10aa4e8 100644 --- a/mm2src/mm2_bitcoin/primitives/src/hash.rs +++ b/mm2src/mm2_bitcoin/primitives/src/hash.rs @@ -1,5 +1,6 @@ //! Fixed-size hashes +use bitcoin_hashes::{sha256d, Hash as ExtHash}; use hex::{FromHex, FromHexError, ToHex}; use std::hash::{Hash, Hasher}; use std::{cmp, fmt, ops, str}; @@ -165,4 +166,7 @@ impl H256 { #[inline] pub fn to_reversed_str(self) -> String { self.reversed().to_string() } + + #[inline] + pub fn to_sha256d(self) -> sha256d::Hash { sha256d::Hash::from_inner(self.take()) } } diff --git a/mm2src/mm2_bitcoin/primitives/src/lib.rs b/mm2src/mm2_bitcoin/primitives/src/lib.rs index 9e07a813c7..78dc7330e7 100644 --- a/mm2src/mm2_bitcoin/primitives/src/lib.rs +++ b/mm2src/mm2_bitcoin/primitives/src/lib.rs @@ -1,6 +1,7 @@ #![allow(clippy::assign_op_pattern)] #![allow(clippy::ptr_offset_with_cast)] +extern crate bitcoin_hashes; extern crate byteorder; extern crate rustc_hex as hex; extern crate uint; diff --git a/mm2src/mm2_bitcoin/rpc/src/v1/types/get_block_response.rs b/mm2src/mm2_bitcoin/rpc/src/v1/types/get_block_response.rs index bebfd32878..ba29704208 100644 --- a/mm2src/mm2_bitcoin/rpc/src/v1/types/get_block_response.rs +++ b/mm2src/mm2_bitcoin/rpc/src/v1/types/get_block_response.rs @@ -124,7 +124,7 @@ mod tests { let block = VerboseBlock::default(); assert_eq!( serde_json::to_string(&block).unwrap(), - r#"{"hash":"0000000000000000000000000000000000000000000000000000000000000000","confirmations":0,"size":0,"strippedsize":0,"weight":0,"height":null,"version":0,"versionHex":"","merkleroot":"0000000000000000000000000000000000000000000000000000000000000000","tx":[],"time":0,"mediantime":null,"nonce":0,"bits":0,"difficulty":0.0,"chainwork":"0","previousblockhash":null,"nextblockhash":null}"# + r#"{"hash":"0000000000000000000000000000000000000000000000000000000000000000","confirmations":0,"size":0,"strippedsize":0,"weight":0,"height":null,"version":0,"versionHex":"","merkleroot":"0000000000000000000000000000000000000000000000000000000000000000","tx":[],"time":0,"mediantime":null,"nonce":0,"bits":0,"difficulty":0.0,"chainwork":"00","previousblockhash":null,"nextblockhash":null}"# ); let block = VerboseBlock { @@ -149,7 +149,7 @@ mod tests { }; assert_eq!( serde_json::to_string(&block).unwrap(), - r#"{"hash":"0100000000000000000000000000000000000000000000000000000000000000","confirmations":-1,"size":500000,"strippedsize":444444,"weight":5236235,"height":3513513,"version":1,"versionHex":"01","merkleroot":"0200000000000000000000000000000000000000000000000000000000000000","tx":["0300000000000000000000000000000000000000000000000000000000000000","0400000000000000000000000000000000000000000000000000000000000000"],"time":111,"mediantime":100,"nonce":124,"bits":13513,"difficulty":555.555,"chainwork":"3","previousblockhash":"0400000000000000000000000000000000000000000000000000000000000000","nextblockhash":"0500000000000000000000000000000000000000000000000000000000000000"}"# + r#"{"hash":"0100000000000000000000000000000000000000000000000000000000000000","confirmations":-1,"size":500000,"strippedsize":444444,"weight":5236235,"height":3513513,"version":1,"versionHex":"01","merkleroot":"0200000000000000000000000000000000000000000000000000000000000000","tx":["0300000000000000000000000000000000000000000000000000000000000000","0400000000000000000000000000000000000000000000000000000000000000"],"time":111,"mediantime":100,"nonce":124,"bits":13513,"difficulty":555.555,"chainwork":"03","previousblockhash":"0400000000000000000000000000000000000000000000000000000000000000","nextblockhash":"0500000000000000000000000000000000000000000000000000000000000000"}"# ); } @@ -197,7 +197,7 @@ mod tests { let verbose_response = GetBlockResponse::Verbose(Box::new(block)); assert_eq!( serde_json::to_string(&verbose_response).unwrap(), - r#"{"hash":"0000000000000000000000000000000000000000000000000000000000000000","confirmations":0,"size":0,"strippedsize":0,"weight":0,"height":null,"version":0,"versionHex":"","merkleroot":"0000000000000000000000000000000000000000000000000000000000000000","tx":[],"time":0,"mediantime":null,"nonce":0,"bits":0,"difficulty":0.0,"chainwork":"0","previousblockhash":null,"nextblockhash":null}"# + r#"{"hash":"0000000000000000000000000000000000000000000000000000000000000000","confirmations":0,"size":0,"strippedsize":0,"weight":0,"height":null,"version":0,"versionHex":"","merkleroot":"0000000000000000000000000000000000000000000000000000000000000000","tx":[],"time":0,"mediantime":null,"nonce":0,"bits":0,"difficulty":0.0,"chainwork":"00","previousblockhash":null,"nextblockhash":null}"# ); } } diff --git a/mm2src/mm2_bitcoin/spv_validation/Cargo.toml b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml index ccf46b61ae..6d3a5e96d6 100644 --- a/mm2src/mm2_bitcoin/spv_validation/Cargo.toml +++ b/mm2src/mm2_bitcoin/spv_validation/Cargo.toml @@ -2,9 +2,13 @@ name = "spv_validation" version = "0.1.0" authors = ["Roman Sztergbaum "] +edition = "2018" [dependencies] +async-trait = "0.1" chain = {path = "../chain"} +derive_more = "0.99" +keys = {path = "../keys"} primitives = { path = "../primitives" } ripemd160 = "0.9.0" rustc-hex = "2" @@ -13,5 +17,7 @@ sha2 = "0.9" test_helpers = { path = "../test_helpers" } [dev-dependencies] +common = { path = "../../common" } +lazy_static = "1.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" \ No newline at end of file diff --git a/mm2src/mm2_bitcoin/spv_validation/src/for_tests/workTestVectors.json b/mm2src/mm2_bitcoin/spv_validation/src/for_tests/workTestVectors.json new file mode 100644 index 0000000000..764b633307 --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/for_tests/workTestVectors.json @@ -0,0 +1,30 @@ +{ + "BTC": [ + { + "height": 2016, + "hex": "010000006397bb6abd4fc521c0d3f6071b5650389f0b4551bc40b4e6b067306900000000ace470aecda9c8818c8fe57688cd2a772b5a57954a00df0420a7dd546b6d2c576b0e7f49ffff001d33f0192f" + }, + { + "height": 604800, + "hex": "000000208e244d2c55bc403caa5d6eaf0f922170e413eb1e02fb02000000000000000000e03b4d9df72d8db232a20bb2ff35c433a99f1467f391f75b5f62180d96f06d6aa4c4d65d3eb215179ef91633" + } + ], + "tBTC": [ + { + "height": 199584, + "hex": "0200000097f2b61897ba2bed756cca30058bcc1c2dfbb4ed0e962f47f749dc03000000006b80079a1eda8071424e294fa56849370e331c8ff7e95034576c9789c8db0fa6da551153ab80011c9bdaca25" + }, + { + "height": 201594, + "hex": "02000000cbed7fd98f1f06e85c47e13ff956533642056be45e7e6b532d4d768f00000000f2680982f333fcc9afa7f9a5e2a84dc54b7fe10605cd187362980b3aa882e9683be21353ab80011c813e1fc0" + }, + { + "height": 201595, + "hex": "02000000ea01a61a2d7420a1b23875e40eb5eb4ca18b378902c8e6384514ad0000000000c0c5a1ae80582b3fe319d8543307fa67befc2a734b8eddb84b1780dfdf11fa2b20e71353ffff001d00805fe0" + }, + { + "height": 201596, + "hex": "02000000303505969a1df329e5fccdf69b847a201772e116e557eb7f119d1a9600000000469267f52f43b8799e72f0726ba2e56432059a8ad02b84d4fff84b9476e95f7716e41353ab80011c168cb471" + } + ] +} \ No newline at end of file diff --git a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs index a91ff91289..6c3c263960 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/helpers_validation.rs @@ -1,50 +1,52 @@ use chain::{BlockHeader, RawBlockHeader, RawHeaderError}; +use derive_more::Display; use primitives::hash::H256; use primitives::U256; use ripemd160::Digest; use serialization::parse_compact_int; use sha2::Sha256; -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Display, PartialEq, Eq, Clone)] pub enum SPVError { - /// Overran a checked read on a slice + #[display(fmt = "Overran a checked read on a slice")] ReadOverrun, - /// Attempted to parse a CompactInt without enough bytes + #[display(fmt = "Attempted to parse a CompactInt without enough bytes")] BadCompactInt, - /// `extract_hash` could not identify the output type. + #[display(fmt = "`extract_hash` could not identify the output type")] MalformattedOutput, - /// Unable to get target from block header + #[display(fmt = "Unable to get target from block header")] UnableToGetTarget, - /// Unable to get block header from network or storage - UnableToGetHeader, - /// Header not exactly 80 bytes. + #[display(fmt = "Unable to get block header from network or storage: {}", _0)] + UnableToGetHeader(String), + #[display(fmt = "Header not exactly 80 bytes")] WrongLengthHeader, - /// Header chain changed difficulties unexpectedly + #[display(fmt = "Header chain changed difficulties unexpectedly")] UnexpectedDifficultyChange, - /// Header does not meet its own difficulty target. + #[display(fmt = "Header does not meet its own difficulty target")] InsufficientWork, - /// Header in chain does not correctly reference parent header. + #[display(fmt = "Header in chain does not correctly reference parent header")] InvalidChain, - /// When validating a `BitcoinHeader`, the `hash` field is not the digest - /// of the raw header. + #[display(fmt = "When validating a `BitcoinHeader`, the `hash` field is not the digest of the raw header")] WrongDigest, - /// When validating a `BitcoinHeader`, the `merkle_root` field does not - /// match the root found in the raw header. + #[display( + fmt = "When validating a `BitcoinHeader`, the `merkle_root` field does not match the root found in the raw header" + )] WrongMerkleRoot, - /// When validating a `BitcoinHeader`, the `prevhash` field does not - /// match the parent hash found in the raw header. + #[display( + fmt = "When validating a `BitcoinHeader`, the `prevhash` field does not match the parent hash found in the raw header" + )] WrongPrevHash, - /// A `vin` (transaction input vector) is malformatted. + #[display(fmt = "A `vin` (transaction input vector) is malformatted")] InvalidVin, - /// A `vout` (transaction output vector) is malformatted or empty. + #[display(fmt = "A `vout` (transaction output vector) is malformatted or empty")] InvalidVout, - /// merkle proof connecting the `tx_id_le` to the `confirming_header`. + #[display(fmt = "merkle proof connecting the `tx_id_le` to the `confirming_header`")] BadMerkleProof, - /// Unable to get merkle tree from network or storage - UnableToGetMerkle, - /// Unable to retrieve block height / block height is zero. - InvalidHeight, - /// Raises during validation loop + #[display(fmt = "Unable to get merkle tree from network or storage: {}", _0)] + UnableToGetMerkle(String), + #[display(fmt = "Unable to retrieve block height / block height is zero: {}", _0)] + InvalidHeight(String), + #[display(fmt = "Raises during validation loop")] Timeout, } diff --git a/mm2src/mm2_bitcoin/spv_validation/src/lib.rs b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs index 1c30005769..d74dc35392 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/lib.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/lib.rs @@ -1,4 +1,6 @@ extern crate chain; +extern crate derive_more; +extern crate keys; extern crate primitives; extern crate ripemd160; extern crate rustc_hex as hex; @@ -12,6 +14,12 @@ pub mod helpers_validation; /// `spv_proof` Contains spv proof validation logic and data structure pub mod spv_proof; +/// `storage` Contains traits that can be implemented to provide the storage needed for spv validation +pub mod storage; + +/// `work` Contains functions that can be used to calculate proof of work difficulty, target, bits, etc... +pub mod work; + #[cfg(test)] pub(crate) mod test_utils { extern crate serde; diff --git a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs index 10df7433dd..646afc0f16 100644 --- a/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs +++ b/mm2src/mm2_bitcoin/spv_validation/src/spv_proof.rs @@ -1,6 +1,6 @@ +use crate::helpers_validation::{merkle_prove, validate_vin, validate_vout, SPVError}; use chain::BlockHeader; use chain::RawBlockHeader; -use helpers_validation::{merkle_prove, validate_vin, validate_vout, SPVError}; use primitives::hash::H256; pub const TRY_SPV_PROOF_INTERVAL: u64 = 10; @@ -68,11 +68,11 @@ impl SPVProof { #[cfg(test)] mod spv_proof_tests { + use crate::spv_proof::SPVProof; use chain::BlockHeader; use chain::RawBlockHeader; use hex::FromHex; use serialization::deserialize; - use spv_proof::SPVProof; #[test] fn test_block_header() { diff --git a/mm2src/mm2_bitcoin/spv_validation/src/storage.rs b/mm2src/mm2_bitcoin/spv_validation/src/storage.rs new file mode 100644 index 0000000000..d516535f79 --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/storage.rs @@ -0,0 +1,82 @@ +use async_trait::async_trait; +use chain::BlockHeader; +use derive_more::Display; +use primitives::hash::H256; +use std::collections::HashMap; + +#[derive(Debug, Display)] +pub enum BlockHeaderStorageError { + #[display(fmt = "Can't add to the storage for {} - reason: {}", coin, reason)] + AddToStorageError { + coin: String, + reason: String, + }, + #[display(fmt = "Can't get from the storage for {} - reason: {}", coin, reason)] + GetFromStorageError { + coin: String, + reason: String, + }, + #[display(fmt = "Can't retrieve the table from the storage for {} - reason: {}", coin, reason)] + CantRetrieveTableError { + coin: String, + reason: String, + }, + #[display(fmt = "Can't query from the storage - query: {} - reason: {}", query, reason)] + QueryError { + query: String, + reason: String, + }, + #[display(fmt = "Can't init from the storage - coin: {} - reason: {}", coin, reason)] + InitializationError { + coin: String, + reason: String, + }, + #[display(fmt = "Can't decode/deserialize from storage for {} - reason: {}", coin, reason)] + DecodeError { + coin: String, + reason: String, + }, + Internal(String), +} + +#[async_trait] +pub trait BlockHeaderStorageOps: Send + Sync + 'static { + /// Initializes collection/tables in storage for a specified coin + async fn init(&self, for_coin: &str) -> Result<(), BlockHeaderStorageError>; + + async fn is_initialized_for(&self, for_coin: &str) -> Result; + + // Adds multiple block headers to the selected coin's header storage + // Should store it as `COIN_HEIGHT=hex_string` + // use this function for headers that comes from `blockchain_block_headers` + async fn add_block_headers_to_storage( + &self, + for_coin: &str, + headers: HashMap, + ) -> Result<(), BlockHeaderStorageError>; + + /// Gets the block header by height from the selected coin's storage as BlockHeader + async fn get_block_header( + &self, + for_coin: &str, + height: u64, + ) -> Result, BlockHeaderStorageError>; + + /// Gets the block header by height from the selected coin's storage as hex + async fn get_block_header_raw( + &self, + for_coin: &str, + height: u64, + ) -> Result, BlockHeaderStorageError>; + + async fn get_last_block_header_with_non_max_bits( + &self, + for_coin: &str, + ) -> Result, BlockHeaderStorageError>; + + async fn get_block_height_by_hash( + &self, + for_coin: &str, + hash: H256, + ) -> Result, BlockHeaderStorageError>; +} diff --git a/mm2src/mm2_bitcoin/spv_validation/src/work.rs b/mm2src/mm2_bitcoin/spv_validation/src/work.rs new file mode 100644 index 0000000000..ed5935006d --- /dev/null +++ b/mm2src/mm2_bitcoin/spv_validation/src/work.rs @@ -0,0 +1,337 @@ +use crate::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; +use chain::{BlockHeader, BlockHeaderBits}; +use derive_more::Display; +use primitives::compact::Compact; +use primitives::U256; +use std::cmp; + +const RETARGETING_FACTOR: u32 = 4; +const TARGET_SPACING_SECONDS: u32 = 10 * 60; +const TARGET_TIMESPAN_SECONDS: u32 = 2 * 7 * 24 * 60 * 60; + +/// The Target number of blocks equals to 2 weeks or 2016 blocks +const RETARGETING_INTERVAL: u32 = TARGET_TIMESPAN_SECONDS / TARGET_SPACING_SECONDS; + +/// The upper and lower bounds for retargeting timespan +const MIN_TIMESPAN: u32 = TARGET_TIMESPAN_SECONDS / RETARGETING_FACTOR; +const MAX_TIMESPAN: u32 = TARGET_TIMESPAN_SECONDS * RETARGETING_FACTOR; + +/// The maximum value for bits corresponding to lowest difficulty of 1 +pub const MAX_BITS_BTC: u32 = 486604799; + +fn is_retarget_height(height: u32) -> bool { height % RETARGETING_INTERVAL == 0 } + +#[derive(Debug, Display)] +pub enum NextBlockBitsError { + #[display(fmt = "Block headers storage error: {}", _0)] + StorageError(BlockHeaderStorageError), + #[display(fmt = "Can't find Block header for {} with height {}", height, coin)] + NoSuchBlockHeader { + coin: String, + height: u64, + }, + #[display(fmt = "Can't find a Block header for {} with no max bits", coin)] + NoBlockHeaderWithNoMaxBits { + coin: String, + }, + Internal(String), +} + +impl From for NextBlockBitsError { + fn from(e: BlockHeaderStorageError) -> Self { NextBlockBitsError::StorageError(e) } +} + +pub enum DifficultyAlgorithm { + BitcoinMainnet, + BitcoinTestnet, +} + +pub async fn next_block_bits( + coin: &str, + current_block_timestamp: u32, + last_block_header: BlockHeader, + last_block_height: u32, + storage: &dyn BlockHeaderStorageOps, + algorithm: DifficultyAlgorithm, +) -> Result { + match algorithm { + DifficultyAlgorithm::BitcoinMainnet => { + btc_mainnet_next_block_bits(coin, last_block_header, last_block_height, storage).await + }, + DifficultyAlgorithm::BitcoinTestnet => { + btc_testnet_next_block_bits( + coin, + current_block_timestamp, + last_block_header, + last_block_height, + storage, + ) + .await + }, + } +} + +fn range_constrain(value: i64, min: i64, max: i64) -> i64 { cmp::min(cmp::max(value, min), max) } + +/// Returns constrained number of seconds since last retarget +fn retarget_timespan(retarget_timestamp: u32, last_timestamp: u32) -> u32 { + // subtract unsigned 32 bit numbers in signed 64 bit space in + // order to prevent underflow before applying the range constraint. + let timespan = last_timestamp as i64 - retarget_timestamp as i64; + range_constrain(timespan, MIN_TIMESPAN as i64, MAX_TIMESPAN as i64) as u32 +} + +async fn btc_retarget_bits( + coin: &str, + height: u32, + last_block_header: BlockHeader, + storage: &dyn BlockHeaderStorageOps, +) -> Result { + let retarget_ref = (height - RETARGETING_INTERVAL).into(); + let retarget_header = + storage + .get_block_header(coin, retarget_ref) + .await? + .ok_or(NextBlockBitsError::NoSuchBlockHeader { + coin: coin.into(), + height: retarget_ref, + })?; + // timestamp of block(height - RETARGETING_INTERVAL) + let retarget_timestamp = retarget_header.time; + // timestamp of last block + let last_timestamp = last_block_header.time; + + let retarget: Compact = last_block_header.bits.into(); + let retarget: U256 = retarget.into(); + let retarget_timespan: U256 = retarget_timespan(retarget_timestamp, last_timestamp).into(); + let retarget: U256 = retarget * retarget_timespan; + let target_timespan_seconds: U256 = TARGET_TIMESPAN_SECONDS.into(); + let retarget = retarget / target_timespan_seconds; + + let max_bits_compact: Compact = MAX_BITS_BTC.into(); + let max_bits: U256 = max_bits_compact.into(); + + if retarget > max_bits { + Ok(BlockHeaderBits::Compact(max_bits_compact)) + } else { + Ok(BlockHeaderBits::Compact(retarget.into())) + } +} + +async fn btc_mainnet_next_block_bits( + coin: &str, + last_block_header: BlockHeader, + last_block_height: u32, + storage: &dyn BlockHeaderStorageOps, +) -> Result { + if last_block_height == 0 { + return Err(NextBlockBitsError::Internal("Last block height can't be zero".into())); + } + + let height = last_block_height + 1; + let last_block_bits = last_block_header.bits.clone(); + + if is_retarget_height(height) { + btc_retarget_bits(coin, height, last_block_header, storage).await + } else { + Ok(last_block_bits) + } +} + +async fn btc_testnet_next_block_bits( + coin: &str, + current_block_timestamp: u32, + last_block_header: BlockHeader, + last_block_height: u32, + storage: &dyn BlockHeaderStorageOps, +) -> Result { + if last_block_height == 0 { + return Err(NextBlockBitsError::Internal("Last block height can't be zero".into())); + } + + let height = last_block_height + 1; + let last_block_bits = last_block_header.bits.clone(); + let max_time_gap = last_block_header.time + 2 * TARGET_SPACING_SECONDS; + let max_bits = BlockHeaderBits::Compact(MAX_BITS_BTC.into()); + + if is_retarget_height(height) { + btc_retarget_bits(coin, height, last_block_header, storage).await + } else if current_block_timestamp > max_time_gap { + Ok(max_bits) + } else if last_block_bits != max_bits { + Ok(last_block_bits.clone()) + } else { + let last_block_header_with_non_max_bits = storage + .get_last_block_header_with_non_max_bits(coin) + .await? + .ok_or(NextBlockBitsError::NoBlockHeaderWithNoMaxBits { coin: coin.into() })?; + Ok(last_block_header_with_non_max_bits.bits) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; + use async_trait::async_trait; + use common::block_on; + use lazy_static::lazy_static; + use primitives::hash::H256; + use serde::Deserialize; + use std::collections::HashMap; + + const BLOCK_HEADERS_STR: &str = include_str!("./for_tests/workTestVectors.json"); + + #[derive(Deserialize)] + struct TestRawHeader { + height: u64, + hex: String, + } + + lazy_static! { + static ref BLOCK_HEADERS_MAP: HashMap> = parse_block_headers(); + } + + fn parse_block_headers() -> HashMap> { serde_json::from_str(BLOCK_HEADERS_STR).unwrap() } + + fn get_block_headers_for_coin(coin: &str) -> HashMap { + BLOCK_HEADERS_MAP + .get(coin) + .unwrap() + .into_iter() + .map(|h| (h.height, h.hex.as_str().into())) + .collect() + } + + struct TestBlockHeadersStorage {} + + #[async_trait] + impl BlockHeaderStorageOps for TestBlockHeadersStorage { + async fn init(&self, _for_coin: &str) -> Result<(), BlockHeaderStorageError> { Ok(()) } + + async fn is_initialized_for(&self, _for_coin: &str) -> Result { Ok(true) } + + async fn add_block_headers_to_storage( + &self, + _for_coin: &str, + _headers: HashMap, + ) -> Result<(), BlockHeaderStorageError> { + Ok(()) + } + + async fn get_block_header( + &self, + for_coin: &str, + height: u64, + ) -> Result, BlockHeaderStorageError> { + Ok(get_block_headers_for_coin(for_coin).get(&height).cloned()) + } + + async fn get_block_header_raw( + &self, + _for_coin: &str, + _height: u64, + ) -> Result, BlockHeaderStorageError> { + Ok(None) + } + + async fn get_last_block_header_with_non_max_bits( + &self, + for_coin: &str, + ) -> Result, BlockHeaderStorageError> { + let mut headers = get_block_headers_for_coin(for_coin); + headers.retain(|_, h| h.bits != BlockHeaderBits::Compact(MAX_BITS_BTC.into())); + let header = headers.into_iter().max_by(|a, b| a.0.cmp(&b.0)); + Ok(header.map(|(_, h)| h)) + } + + async fn get_block_height_by_hash( + &self, + _for_coin: &str, + _hash: H256, + ) -> Result, BlockHeaderStorageError> { + Ok(None) + } + } + + #[test] + fn test_btc_mainnet_next_block_bits() { + let storage = TestBlockHeadersStorage {}; + + let last_header: BlockHeader = "000000201d758432ecd495a2177b44d3fe6c22af183461a0b9ea0d0000000000000000008283a1dfa795d9b68bd8c18601e443368265072cbf8c76bfe58de46edd303798035de95d3eb2151756fdb0e8".into(); + + let next_block_bits = block_on(btc_mainnet_next_block_bits("BTC", last_header, 606815, &storage)).unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(387308498.into())); + + // check that bits for very early blocks that didn't change difficulty because of low hashrate is calculated correctly. + let last_header: BlockHeader = "010000000d9c8c96715756b619116cc2160937fb26c655a2f8e28e3a0aff59c0000000007676252e8434de408ea31920d986aba297bd6f7c6f20756be08748713f7c135962719449ffff001df8c1cb01".into(); + + let next_block_bits = block_on(btc_mainnet_next_block_bits("BTC", last_header, 4031, &storage)).unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(486604799.into())); + + // check that bits stay the same when the next block is not a retarget block + // https://live.blockcypher.com/btc/block/00000000000000000002622f52b6afe70a5bb139c788e67f221ffc67a762a1e0/ + let last_header: BlockHeader = "00e0ff2f44d953fe12a047129bbc7164668c6d96f3e7a553528b02000000000000000000d0b950384cd23ab0854d1c8f23fa7a97411a6ffd92347c0a3aea4466621e4093ec09c762afa7091705dad220".into(); + + let next_block_bits = block_on(btc_mainnet_next_block_bits("BTC", last_header, 744014, &storage)).unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(386508719.into())); + } + + #[test] + fn test_btc_testnet_next_block_bits() { + let storage = TestBlockHeadersStorage {}; + + // https://live.blockcypher.com/btc-testnet/block/000000000057db3806384e2ec1b02b2c86bd928206ff8dff98f54d616b7fa5f2/ + let current_header: BlockHeader = "02000000303505969a1df329e5fccdf69b847a201772e116e557eb7f119d1a9600000000469267f52f43b8799e72f0726ba2e56432059a8ad02b84d4fff84b9476e95f7716e41353ab80011c168cb471".into(); + // https://live.blockcypher.com/btc-testnet/block/00000000961a9d117feb57e516e17217207a849bf6cdfce529f31d9a96053530/ + let last_header: BlockHeader = "02000000ea01a61a2d7420a1b23875e40eb5eb4ca18b378902c8e6384514ad0000000000c0c5a1ae80582b3fe319d8543307fa67befc2a734b8eddb84b1780dfdf11fa2b20e71353ffff001d00805fe0".into(); + + let next_block_bits = block_on(btc_testnet_next_block_bits( + "tBTC", + current_header.time, + last_header, + 201595, + &storage, + )) + .unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(469860523.into())); + + // https://live.blockcypher.com/btc-testnet/block/00000000961a9d117feb57e516e17217207a849bf6cdfce529f31d9a96053530/ + let current_header: BlockHeader = "02000000ea01a61a2d7420a1b23875e40eb5eb4ca18b378902c8e6384514ad0000000000c0c5a1ae80582b3fe319d8543307fa67befc2a734b8eddb84b1780dfdf11fa2b20e71353ffff001d00805fe0".into(); + // https://live.blockcypher.com/btc-testnet/block/0000000000ad144538e6c80289378ba14cebb50ee47538b2a120742d1aa601ea/ + let last_header: BlockHeader = "02000000cbed7fd98f1f06e85c47e13ff956533642056be45e7e6b532d4d768f00000000f2680982f333fcc9afa7f9a5e2a84dc54b7fe10605cd187362980b3aa882e9683be21353ab80011c813e1fc0".into(); + + let next_block_bits = block_on(btc_testnet_next_block_bits( + "tBTC", + current_header.time, + last_header, + 201594, + &storage, + )) + .unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(486604799.into())); + + // test testnet retarget bits + + // https://live.blockcypher.com/btc-testnet/block/0000000000376bb71314321c45de3015fe958543afcbada242a3b1b072498e38/ + let current_header: BlockHeader = "02000000ee689e4dcdc3c7dac591b98e1e4dc83aae03ff9fb9d469d704a64c0100000000bfffaded2a67821eb5729b362d613747e898d08d6c83b5704646c26c13146f4c6de91353c02a601b3a817f87".into(); + // https://live.blockcypher.com/btc-testnet/block/00000000014ca604d769d4b99fff03ae3ac84d1e8eb991c5dac7c3cd4d9e68ee/ + let last_header: BlockHeader = "02000000a9dccfcf372d6ce6ae784786ea94d20ce174e093520d779348e5a9000000000077c037863a0134ac05a8c19d258d6c03c225043a08687c90813e8352a144d68035e81353ab80011ca71f3849".into(); + + let next_block_bits = block_on(btc_testnet_next_block_bits( + "tBTC", + current_header.time, + last_header, + 201599, + &storage, + )) + .unwrap(); + + assert_eq!(next_block_bits, BlockHeaderBits::Compact(459287232.into())); + } +}