(WIP) AuxPoW: Parallel blockchain sync

electrum/constants.py

@@ -50,7 +50,7 @@ class AbstractNet:
 
     @classmethod
     def max_checkpoint(cls) -> int:
-        return max(0, len(cls.CHECKPOINTS) * 2016 - 1)
+        return cls.CHECKPOINTS['height']
 
     @classmethod
     def rev_genesis_bytes(cls) -> bytes:
@@ -67,7 +67,10 @@ class BitcoinMainnet(AbstractNet):
     GENESIS = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"
     DEFAULT_PORTS = {'t': '50001', 's': '50002'}
     DEFAULT_SERVERS = read_json('servers.json', {})
-    CHECKPOINTS = read_json('checkpoints.json', [])
+    # To generate this JSON file, connect to a trusted server, and then run
+    # this from the console:
+    # network.run_from_another_thread(network.interface.export_purported_checkpoints(height, path))
+    CHECKPOINTS = read_json('checkpoints.json', {'height': 0})
     BLOCK_HEIGHT_FIRST_LIGHTNING_CHANNELS = 497000
 
     XPRV_HEADERS = {
@@ -104,7 +107,7 @@ class BitcoinTestnet(AbstractNet):
     GENESIS = "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"
     DEFAULT_PORTS = {'t': '51001', 's': '51002'}
     DEFAULT_SERVERS = read_json('servers_testnet.json', {})
-    CHECKPOINTS = read_json('checkpoints_testnet.json', [])
+    CHECKPOINTS = read_json('checkpoints_testnet.json', {'height': 0})
 
     XPRV_HEADERS = {
         'standard':    0x04358394,  # tprv
@@ -135,7 +138,7 @@ class BitcoinRegtest(BitcoinTestnet):
     SEGWIT_HRP = "bcrt"
     GENESIS = "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"
     DEFAULT_SERVERS = read_json('servers_regtest.json', {})
-    CHECKPOINTS = []
+    CHECKPOINTS = {'height': 0}
     LN_DNS_SEEDS = []
 
 
@@ -147,7 +150,7 @@ class BitcoinSimnet(BitcoinTestnet):
     SEGWIT_HRP = "sb"
     GENESIS = "683e86bd5c6d110d91b94b97137ba6bfe02dbbdb8e3dff722a669b5d69d77af6"
     DEFAULT_SERVERS = read_json('servers_regtest.json', {})
-    CHECKPOINTS = []
+    CHECKPOINTS = {'height': 0}
     LN_DNS_SEEDS = []
 
 

electrum/network.py

@@ -251,6 +251,7 @@ def __init__(self, config: SimpleConfig):
         self.logger.info(f"blockchains {list(map(lambda b: b.forkpoint, blockchain.blockchains.values()))}")
         self._blockchain_preferred_block = self.config.get('blockchain_preferred_block', None)  # type: Optional[Dict]
         self._blockchain = blockchain.get_best_chain()
+        self.pending_chunks = {}
         # Server for addresses and transactions
         self.default_server = self.config.get('server', None)
         # Sanitize default server
@@ -823,7 +824,7 @@ def check_interface_against_healthy_spread_of_connected_servers(self, iface_to_c
     async def _init_headers_file(self):
         b = blockchain.get_best_chain()
         filename = b.path()
-        length = HEADER_SIZE * len(constants.net.CHECKPOINTS) * 2016
+        length = HEADER_SIZE * constants.net.max_checkpoint()
         if not os.path.exists(filename) or os.path.getsize(filename) < length:
             with open(filename, 'wb') as f:
                 if length > 0:
@@ -1137,14 +1138,6 @@ async def follow_chain_given_server(self, server_str: str) -> None:
     def get_local_height(self):
         return self.blockchain().height()
 
-    def export_checkpoints(self, path):
-        """Run manually to generate blockchain checkpoints.
-        Kept for console use only.
-        """
-        cp = self.blockchain().get_checkpoints()
-        with open(path, 'w', encoding='utf-8') as f:
-            f.write(json.dumps(cp, indent=4))
-
     async def _start(self):
         assert not self.main_taskgroup
         self.main_taskgroup = main_taskgroup = SilentTaskGroup()

electrum/tests/test_blockchain.py

@@ -4,7 +4,7 @@
 
 from electrum import constants, blockchain
 from electrum.simple_config import SimpleConfig
-from electrum.blockchain import Blockchain, deserialize_pure_header, hash_header
+from electrum.blockchain import Blockchain, MissingHeader, deserialize_pure_header, hash_header
 from electrum.util import bh2u, bfh, make_dir
 
 from . import ElectrumTestCase
@@ -45,23 +45,18 @@ class TestBlockchain(ElectrumTestCase):
     #                           /
     # A <- B <- C <- D <- E <- F <- O <- P <- Q <- R <- S <- T <- U
 
-    @classmethod
-    def setUpClass(cls):
-        super().setUpClass()
-        constants.set_regtest()
-
-    @classmethod
-    def tearDownClass(cls):
-        super().tearDownClass()
-        constants.set_mainnet()
-
     def setUp(self):
         super().setUp()
+        constants.set_regtest()
         self.data_dir = self.electrum_path
         make_dir(os.path.join(self.data_dir, 'forks'))
         self.config = SimpleConfig({'electrum_path': self.data_dir})
         blockchain.blockchains = {}
 
+    def tearDown(self):
+        super().tearDown()
+        constants.set_mainnet()
+
     def _append_header(self, chain: Blockchain, header: dict):
         self.assertTrue(chain.can_connect(header))
         chain.save_header(header)
@@ -336,6 +331,21 @@ def test_doing_multiple_swaps_after_single_new_header(self):
         for b in (chain_u, chain_l, chain_z):
             self.assertTrue(all([b.can_connect(b.read_header(i), False) for i in range(b.height())]))
 
+    def test_mainnet_get_chainwork(self):
+        constants.set_mainnet()
+        blockchain.blockchains[constants.net.GENESIS] = chain_u = Blockchain(
+            config=self.config, forkpoint=0, parent=None,
+            forkpoint_hash=constants.net.GENESIS, prev_hash=None)
+        open(chain_u.path(), 'w+').close()
+
+        # Try a variety of checkpoint heights relative to the chunk boundary
+        for height_offset in range(2016):
+            constants.net.CHECKPOINTS['height']+=1
+
+            chain_u.get_chainwork(constants.net.max_checkpoint())
+            with self.assertRaises(MissingHeader):
+                chain_u.get_chainwork(constants.net.max_checkpoint() + 4032)
+
 
 class TestVerifyHeader(ElectrumTestCase):
 

electrum/tests/test_network.py

@@ -18,6 +18,7 @@ async def spawn(self, x): return
 class MockNetwork:
     main_taskgroup = MockTaskGroup()
     asyncio_loop = asyncio.get_event_loop()
+    bhi_lock = asyncio.Lock()
 
 class MockInterface(Interface):
     def __init__(self, config):
@@ -30,13 +31,13 @@ def __init__(self, config):
                                                 parent=None, forkpoint_hash=constants.net.GENESIS, prev_hash=None)
         self.tip = 12
         self.blockchain._size = self.tip + 1
-    async def get_block_header(self, height, assert_mode):
+    async def get_block_header(self, height, assert_mode, must_provide_proof=False):
         assert self.q.qsize() > 0, (height, assert_mode)
         item = await self.q.get()
         print("step with height", height, item)
         assert item['block_height'] == height, (item['block_height'], height)
         assert assert_mode in item['mock'], (assert_mode, item)
-        return item
+        return item, False
 
 class TestNetwork(ElectrumTestCase):
 

electrum/verifier.py

@@ -30,7 +30,7 @@
 from .crypto import sha256d
 from .bitcoin import hash_decode, hash_encode
 from .transaction import Transaction
-from .blockchain import hash_header
+from .blockchain import MerkleVerificationFailure, hash_header, hash_merkle_root
 from .interface import GracefulDisconnect
 from .network import UntrustedServerReturnedError
 from . import constants
@@ -40,10 +40,8 @@
     from .address_synchronizer import AddressSynchronizer
 
 
-class MerkleVerificationFailure(Exception): pass
 class MissingBlockHeader(MerkleVerificationFailure): pass
 class MerkleRootMismatch(MerkleVerificationFailure): pass
-class InnerNodeOfSpvProofIsValidTx(MerkleVerificationFailure): pass
 
 
 class SPV(NetworkJobOnDefaultServer):
@@ -134,42 +132,6 @@ async def _request_and_verify_single_proof(self, tx_hash, tx_height):
                               header_hash=header_hash)
         self.wallet.add_verified_tx(tx_hash, tx_info)
 
-    @classmethod
-    def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int):
-        """Return calculated merkle root."""
-        try:
-            h = hash_decode(tx_hash)
-            merkle_branch_bytes = [hash_decode(item) for item in merkle_branch]
-            leaf_pos_in_tree = int(leaf_pos_in_tree)  # raise if invalid
-        except Exception as e:
-            raise MerkleVerificationFailure(e)
-        if leaf_pos_in_tree < 0:
-            raise MerkleVerificationFailure('leaf_pos_in_tree must be non-negative')
-        index = leaf_pos_in_tree
-        for item in merkle_branch_bytes:
-            if len(item) != 32:
-                raise MerkleVerificationFailure('all merkle branch items have to 32 bytes long')
-            h = sha256d(item + h) if (index & 1) else sha256d(h + item)
-            index >>= 1
-            cls._raise_if_valid_tx(bh2u(h))
-        if index != 0:
-            raise MerkleVerificationFailure(f'leaf_pos_in_tree too large for branch')
-        return hash_encode(h)
-
-    @classmethod
-    def _raise_if_valid_tx(cls, raw_tx: str):
-        # If an inner node of the merkle proof is also a valid tx, chances are, this is an attack.
-        # https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-June/016105.html
-        # https://lists.linuxfoundation.org/pipermail/bitcoin-dev/attachments/20180609/9f4f5b1f/attachment-0001.pdf
-        # https://bitcoin.stackexchange.com/questions/76121/how-is-the-leaf-node-weakness-in-merkle-trees-exploitable/76122#76122
-        tx = Transaction(raw_tx)
-        try:
-            tx.deserialize()
-        except:
-            pass
-        else:
-            raise InnerNodeOfSpvProofIsValidTx()
-
     async def _maybe_undo_verifications(self):
         old_chain = self.blockchain
         cur_chain = self.network.blockchain()
@@ -199,7 +161,7 @@ def verify_tx_is_in_block(tx_hash: str, merkle_branch: Sequence[str],
                                  .format(tx_hash, block_height))
     if len(merkle_branch) > 30:
         raise MerkleVerificationFailure(f"merkle branch too long: {len(merkle_branch)}")
-    calc_merkle_root = SPV.hash_merkle_root(merkle_branch, tx_hash, leaf_pos_in_tree)
+    calc_merkle_root = hash_merkle_root(merkle_branch, tx_hash, leaf_pos_in_tree)
     if block_header.get('merkle_root') != calc_merkle_root:
         raise MerkleRootMismatch("merkle verification failed for {} ({} != {})".format(
             tx_hash, block_header.get('merkle_root'), calc_merkle_root))