From 336b5db96475e222e969019ff4afe6323af4e147 Mon Sep 17 00:00:00 2001 From: shamardy Date: Thu, 31 Jul 2025 14:21:31 +0300 Subject: [PATCH 1/5] fix(utxo): Correct block header deserialization for AuxPow and KAWPOW coins Swaps for coins like NMC, CHTA, and XEC were failing with an `UnexpectedEnd` error during MTP steps. This was due to overly broad conditional logic in `BlockHeader::deserialize` which misinterpreted block header structures based on version numbers. This commit corrects the logic in two ways: 1. The check for AuxPow headers is changed from a hardcoded list of versions to the correct bitmask check (`version & (1 << 8)`). This is the proper way to detect AuxPow and makes the logic robust against base version changes. 2. KAWPOW header parsing is now specialized to apply only to Ravencoin. A `CoinVariant::RVN` is introduced, and the deserializer now checks for this variant before attempting to read RVN-specific fields. This prevents conflicts with other coins (like CHTA) that use the same version number but have a different header structure. A utility test for scanning and debugging block headers has also been added to assist with future diagnostics. --- mm2src/coins/utxo/utxo_tests.rs | 92 +++++++++++++++++++ mm2src/mm2_bitcoin/chain/src/block_header.rs | 36 +++++--- .../mm2_bitcoin/serialization/src/reader.rs | 8 +- 3 files changed, 123 insertions(+), 13 deletions(-) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 9064fe54b3..e53bacb54e 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -5632,3 +5632,95 @@ fn test_electrum_v14_block_hash() { // Verify V14 header produces the same hash as our verified BlockHeader implementation assert_eq!(hash, headers[0].hash().into()); } + +/// A utility test for debugging block header deserialization issues for any UTXO-based coin. +/// This test is ignored by default and must be run explicitly. +/// +/// It scans a range of block heights, fetching headers in chunks. For each chunk, it reads +/// headers one by one from the data stream. +/// +/// If it encounters a header that fails to parse, it will panic and print detailed information, +/// including the exact block height that failed and the raw hex of the entire chunk for context. +/// +/// # How to Use: +/// 1. Modify the constants in the `CONFIGURATION` section below. +/// 2. Run the test with the `--ignored` flag: `cargo test -- --ignored test_scan_and_deserialize_block_headers` +/// +/// # Debugging Note: +/// If a header at height `N` fails, the error might be caused by the deserializer reading +/// more data than expected from the header at height `N-1`. The full chunk hex provided +/// in the panic message is essential for debugging this scenario. +#[test] +#[ignore = "This is a utility test for debugging header deserialization and must be run explicitly"] +fn test_scan_and_deserialize_block_headers() { + // ========================== CONFIGURATION ========================== + /// The ticker of the coin to test (e.g., "NMC", "CHTA", "RVN"). + const COIN_TICKER: &str = "NMC"; + /// A list of active Electrum servers for the specified coin. + const ELECTRUM_URLS: &[&str] = &["nmc2.bitcoins.sk:57001", "nmc2.bitcoins.sk:57002"]; + /// The block height to start scanning from. + const START_HEIGHT: u64 = 701614; + /// The block height to stop scanning at. Set to `None` to scan to the tip of the chain. + const END_HEIGHT: Option = Some(701616); + /// The number of headers to fetch in a single RPC call. + const CHUNK_SIZE: u64 = 100; + // =================================================================== + + let client = electrum_client_for_test(ELECTRUM_URLS); + let mut current_height = START_HEIGHT; + + loop { + let mut num_to_fetch = CHUNK_SIZE; + if let Some(end_h) = END_HEIGHT { + if current_height > end_h { + println!("Reached configured end height of {}. Scan complete.", end_h); + break; + } + let remaining = end_h.saturating_sub(current_height) + 1; + num_to_fetch = num_to_fetch.min(remaining); + } + + if num_to_fetch == 0 { + break; + } + + println!("Fetching {} headers from height {}", num_to_fetch, current_height); + let headers_res = + block_on_f01(client.blockchain_block_headers(current_height, NonZeroU64::new(num_to_fetch).unwrap())) + .expect("Failed to get block headers"); + + if headers_res.count == 0 { + println!("Reached the end of the chain. No bad header found."); + break; + } + + // This is the correct approach, inspired by your original test. + // We create a single reader for the entire raw byte stream of concatenated headers. + let raw_chunk_bytes = &headers_res.hex.0; + let mut reader = Reader::new_with_coin_variant(raw_chunk_bytes, COIN_TICKER.into()); + + // We loop exactly `count` times, reading one header in each iteration. + // The `read` method will correctly consume a variable number of bytes depending on the header's content. + for i in 0..headers_res.count { + let block_height_of_header = current_height + i; + + if let Err(e) = reader.read::() { + // If a read fails, we've found the problematic header. + // We panic with all the necessary context for debugging. + let chunk_hex_str = hex::encode(raw_chunk_bytes); + panic!( + "\n\n!!! Deserialization failed on header index {} (block height: {}) within the chunk starting at {} !!!\n\ + Raw Chunk Hex: {}\n\ + Deserialization Error: {:?}\n\n", + i, block_height_of_header, current_height, chunk_hex_str, e + ); + } + } + + // If the loop completes, the entire chunk was successfully parsed. + println!("Successfully deserialized chunk starting at height {}.", current_height); + current_height += headers_res.count; + } + + println!("Scan finished successfully. No bad headers found in the specified range."); +} diff --git a/mm2src/mm2_bitcoin/chain/src/block_header.rs b/mm2src/mm2_bitcoin/chain/src/block_header.rs index e4fce6103f..db6737e715 100644 --- a/mm2src/mm2_bitcoin/chain/src/block_header.rs +++ b/mm2src/mm2_bitcoin/chain/src/block_header.rs @@ -61,9 +61,8 @@ impl From for Compact { } } -const AUX_POW_VERSION_DOGE: u32 = 6422788; -const AUX_POW_VERSION_NMC: u32 = 65796; -const AUX_POW_VERSION_SYS: u32 = 537919744; +// The version number used by AuxPow-enabled coins is a bitmask. +const AUXPOW_VERSION_FLAG: u32 = 1 << 8; const MTP_POW_VERSION: u32 = 0x20001000u32; const PROG_POW_SWITCH_TIME: u32 = 1635228000; const BIP9_NO_SOFT_FORK_BLOCK_HEADER_VERSION: u32 = 536870912; @@ -242,7 +241,7 @@ impl Deserializable for BlockHeader { }; let nonce = if is_zcash { BlockHeaderNonce::H256(reader.read()?) - } else if (version == KAWPOW_VERSION && !reader.coin_variant().is_btc()) + } else if (version == KAWPOW_VERSION && reader.coin_variant().is_rvn()) || version == MTP_POW_VERSION && time >= PROG_POW_SWITCH_TIME { BlockHeaderNonce::U32(0) @@ -252,10 +251,7 @@ impl Deserializable for BlockHeader { let solution = if is_zcash { Some(reader.read_list()?) } else { None }; // https://en.bitcoin.it/wiki/Merged_mining_specification#Merged_mining_coinbase - let aux_pow = if matches!( - version, - AUX_POW_VERSION_DOGE | AUX_POW_VERSION_SYS | AUX_POW_VERSION_NMC - ) { + let aux_pow = if (version & AUXPOW_VERSION_FLAG) != 0 { let coinbase_tx = deserialize_tx(reader, TxType::StandardWithWitness)?; let parent_block_hash = reader.read()?; let coinbase_branch = reader.read()?; @@ -297,7 +293,7 @@ impl Deserializable for BlockHeader { }; // https://github.com/RavenProject/Ravencoin/blob/61c790447a5afe150d9892705ac421d595a2df60/src/primitives/block.h#L67 - let (n_height, n_nonce_u64, mix_hash) = if version == KAWPOW_VERSION && !reader.coin_variant().is_btc() { + let (n_height, n_nonce_u64, mix_hash) = if version == KAWPOW_VERSION && reader.coin_variant().is_rvn() { (Some(reader.read()?), Some(reader.read()?), Some(reader.read()?)) } else { (None, None, None) @@ -387,13 +383,15 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] use super::ExtBlockHeader; use block_header::{ - BlockHeader, BlockHeaderBits, BlockHeaderNonce, AUX_POW_VERSION_DOGE, AUX_POW_VERSION_NMC, AUX_POW_VERSION_SYS, - BIP9_NO_SOFT_FORK_BLOCK_HEADER_VERSION, KAWPOW_VERSION, MTP_POW_VERSION, PROG_POW_SWITCH_TIME, + BlockHeader, BlockHeaderBits, BlockHeaderNonce, BIP9_NO_SOFT_FORK_BLOCK_HEADER_VERSION, KAWPOW_VERSION, + MTP_POW_VERSION, PROG_POW_SWITCH_TIME, }; use hex::FromHex; use primitives::bytes::Bytes; use ser::{deserialize, serialize, serialize_list, CoinVariant, Error as ReaderError, Reader, Stream}; + const AUX_POW_VERSION_DOGE: u32 = 6422788; + #[test] fn test_block_header_stream() { let block_header = BlockHeader { @@ -1055,6 +1053,7 @@ mod tests { #[test] fn test_nmc_block_headers_serde_11() { + const AUX_POW_VERSION_NMC: u32 = 65796; // NMC block headers // start - #622807 // end - #622796 @@ -1569,6 +1568,7 @@ mod tests { #[test] fn test_sys_block_headers_serde_11() { + const AUX_POW_VERSION_SYS: u32 = 537919744; let headers_bytes: &[u8] = &[ 11, 0, 1, 16, 32, 224, 75, 244, 161, 185, 248, 38, 215, 191, 60, 214, 46, 170, 129, 104, 51, 104, 181, 69, 119, 171, 121, 183, 144, 38, 57, 67, 7, 99, 24, 201, 157, 180, 182, 0, 37, 120, 169, 194, 158, 75, 173, @@ -2468,7 +2468,7 @@ mod tests { 252, 71, 214, 56, 220, 173, 79, 220, 196, 15, 211, ]; - let mut reader = Reader::new(headers_bytes); + let mut reader = Reader::new_with_coin_variant(headers_bytes, CoinVariant::RVN); let headers = reader.read_list::().unwrap(); for header in headers.iter() { assert_eq!(header.version, KAWPOW_VERSION); @@ -2884,4 +2884,16 @@ mod tests { let serialized = serialize(&header); assert_eq!(serialized.take(), header_bytes); } + + #[test] + fn test_chta_kawpow_version_header() { + let header_hex = "00000030f0aceae7f05f5951ec0a6adae323f6e77bcd28beda092749e30800000000000072c4e8c753c4694ec6152fe97c73f72baf5b58176ca05adc7060989267b4816265ae8268cada0a1afcabfdbd"; + let header_bytes: Vec = header_hex.from_hex().unwrap(); + let header: BlockHeader = deserialize(header_bytes.as_slice()).unwrap(); + + assert_eq!(header.version, KAWPOW_VERSION); + + let serialized = serialize(&header); + assert_eq!(serialized.take(), header_bytes); + } } diff --git a/mm2src/mm2_bitcoin/serialization/src/reader.rs b/mm2src/mm2_bitcoin/serialization/src/reader.rs index aea0ca6f79..9c9d3feaff 100644 --- a/mm2src/mm2_bitcoin/serialization/src/reader.rs +++ b/mm2src/mm2_bitcoin/serialization/src/reader.rs @@ -64,6 +64,7 @@ pub enum CoinVariant { RICK, /// Same reason as RICK. MORTY, + RVN, } impl CoinVariant { @@ -82,6 +83,9 @@ impl CoinVariant { pub fn is_kmd_assetchain(&self) -> bool { matches!(self, CoinVariant::RICK | CoinVariant::MORTY) } + pub fn is_rvn(&self) -> bool { + matches!(self, CoinVariant::RVN) + } } fn ticker_matches(ticker: &str, with: &str) -> bool { @@ -103,8 +107,10 @@ impl From<&str> for CoinVariant { t if ticker_matches(t, "PPC") => CoinVariant::PPC, // "RICK" t if ticker_matches(t, "RICK") => CoinVariant::RICK, - // "MORTY + // "MORTY" t if ticker_matches(t, "MORTY") => CoinVariant::MORTY, + // "RVN" + t if ticker_matches(t, "RVN") => CoinVariant::RVN, _ => CoinVariant::Standard, } } From d7875619e227d1fae745c00d43d3062ff2a2238e Mon Sep 17 00:00:00 2001 From: shamardy Date: Thu, 31 Jul 2025 14:29:47 +0300 Subject: [PATCH 2/5] fix clippy --- mm2src/coins/utxo/utxo_tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 5800303521..4224445f0f 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -5747,7 +5747,7 @@ fn test_scan_and_deserialize_block_headers() { let mut num_to_fetch = CHUNK_SIZE; if let Some(end_h) = END_HEIGHT { if current_height > end_h { - println!("Reached configured end height of {}. Scan complete.", end_h); + println!("Reached configured end height of {end_h}. Scan complete."); break; } let remaining = end_h.saturating_sub(current_height) + 1; @@ -5758,7 +5758,7 @@ fn test_scan_and_deserialize_block_headers() { break; } - println!("Fetching {} headers from height {}", num_to_fetch, current_height); + println!("Fetching {num_to_fetch} headers from height {current_height}"); let headers_res = block_on_f01(client.blockchain_block_headers(current_height, NonZeroU64::new(num_to_fetch).unwrap())) .expect("Failed to get block headers"); @@ -5792,7 +5792,7 @@ fn test_scan_and_deserialize_block_headers() { } // If the loop completes, the entire chunk was successfully parsed. - println!("Successfully deserialized chunk starting at height {}.", current_height); + println!("Successfully deserialized chunk starting at height {current_height}."); current_height += headers_res.count; } From c17069cbe4c2abbd641d068cc4d5916619ba0aac Mon Sep 17 00:00:00 2001 From: shamardy Date: Sat, 2 Aug 2025 15:21:20 +0300 Subject: [PATCH 3/5] apply suggestions from the @mariocynicys review. - In `utxo_tests.rs`: - Remove an unreachable `break` condition from the header scanning loop. - Reformat the panic message in the `test_scan_and_deserialize_block_headers` utility to improve terminal readability by reordering and removing extra indentation. - In `block_header.rs`: - Add parentheses to a conditional in the `BlockHeader` deserialization logic to enhance code clarity and consistency. --- mm2src/coins/utxo/utxo_tests.rs | 10 +++------- mm2src/mm2_bitcoin/chain/src/block_header.rs | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 4224445f0f..7bc064972b 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -5754,10 +5754,6 @@ fn test_scan_and_deserialize_block_headers() { num_to_fetch = num_to_fetch.min(remaining); } - if num_to_fetch == 0 { - break; - } - println!("Fetching {num_to_fetch} headers from height {current_height}"); let headers_res = block_on_f01(client.blockchain_block_headers(current_height, NonZeroU64::new(num_to_fetch).unwrap())) @@ -5784,9 +5780,9 @@ fn test_scan_and_deserialize_block_headers() { let chunk_hex_str = hex::encode(raw_chunk_bytes); panic!( "\n\n!!! Deserialization failed on header index {} (block height: {}) within the chunk starting at {} !!!\n\ - Raw Chunk Hex: {}\n\ - Deserialization Error: {:?}\n\n", - i, block_height_of_header, current_height, chunk_hex_str, e + Deserialization Error: {:?}\n\ + Raw Chunk Hex: {}\n\n", + i, block_height_of_header, current_height, e, chunk_hex_str ); } } diff --git a/mm2src/mm2_bitcoin/chain/src/block_header.rs b/mm2src/mm2_bitcoin/chain/src/block_header.rs index db6737e715..0730fd125d 100644 --- a/mm2src/mm2_bitcoin/chain/src/block_header.rs +++ b/mm2src/mm2_bitcoin/chain/src/block_header.rs @@ -242,7 +242,7 @@ impl Deserializable for BlockHeader { let nonce = if is_zcash { BlockHeaderNonce::H256(reader.read()?) } else if (version == KAWPOW_VERSION && reader.coin_variant().is_rvn()) - || version == MTP_POW_VERSION && time >= PROG_POW_SWITCH_TIME + || (version == MTP_POW_VERSION && time >= PROG_POW_SWITCH_TIME) { BlockHeaderNonce::U32(0) } else { From 50c566b3e53ba44bc5428d6a569ea452ad14892f Mon Sep 17 00:00:00 2001 From: shamardy Date: Mon, 4 Aug 2025 14:34:19 +0300 Subject: [PATCH 4/5] remove indentation one more time --- mm2src/coins/utxo/utxo_tests.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 7bc064972b..dcc5117429 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -5778,12 +5778,7 @@ fn test_scan_and_deserialize_block_headers() { // If a read fails, we've found the problematic header. // We panic with all the necessary context for debugging. let chunk_hex_str = hex::encode(raw_chunk_bytes); - panic!( - "\n\n!!! Deserialization failed on header index {} (block height: {}) within the chunk starting at {} !!!\n\ - Deserialization Error: {:?}\n\ - Raw Chunk Hex: {}\n\n", - i, block_height_of_header, current_height, e, chunk_hex_str - ); + panic!("\n\n!!! Deserialization failed on header index {} (block height: {}) within the chunk starting at {} !!!\nDeserialization Error: {:?}\nRaw Chunk Hex: {}\n\n", i, block_height_of_header, current_height, e, chunk_hex_str); } } From 228426834600e90b0d72525dfdd831c1e10d3892 Mon Sep 17 00:00:00 2001 From: shamardy Date: Mon, 4 Aug 2025 15:04:57 +0300 Subject: [PATCH 5/5] fix failing `rvn_mtp` test --- mm2src/coins/utxo/utxo_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index dcc5117429..161401b8e3 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -3277,8 +3277,8 @@ fn rvn_mtp() { "electrum2.cipig.net:10051", "electrum3.cipig.net:10051", ]); - let mtp = block_on_f01(electrum.get_median_time_past(1968120, NonZeroU64::new(11).unwrap(), CoinVariant::Standard)) - .unwrap(); + let mtp = + block_on_f01(electrum.get_median_time_past(1968120, NonZeroU64::new(11).unwrap(), CoinVariant::RVN)).unwrap(); assert_eq!(mtp, 1633946264); }