From 641d162b1a8f567b9951f14efab560adcb422f7b Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Wed, 7 Jan 2026 23:35:04 -0300 Subject: [PATCH 1/4] feat(p2P): Add new whitelist policy Co-authored-by: LFRezende --- .gitignore | 3 + hathor/builder/builder.py | 25 +- hathor/conf/mainnet.py | 1 - hathor/manager.py | 22 - hathor/p2p/manager.py | 113 +- hathor/p2p/resources/status.py | 7 +- hathor/p2p/states/hello.py | 4 +- hathor/p2p/states/peer_id.py | 35 +- hathor/p2p/sync_version.py | 9 +- hathor/p2p/utils.py | 26 +- hathor/p2p/whitelist/__init__.py | 50 + hathor/p2p/whitelist/factory.py | 62 + hathor/p2p/whitelist/file_whitelist.py | 80 ++ hathor/p2p/whitelist/parsing.py | 118 ++ hathor/p2p/whitelist/peers_whitelist.py | 177 +++ hathor/p2p/whitelist/url_whitelist.py | 123 ++ hathor/sysctl/p2p/manager.py | 106 +- hathor_cli/builder.py | 14 +- hathor_cli/run_node.py | 5 + hathor_cli/run_node_args.py | 2 + .../invalid_byte_hathor_settings_fixture.yml | 2 +- ...valid_features_hathor_settings_fixture.yml | 2 +- .../missing_hathor_settings_fixture.yml | 1 - .../valid_hathor_settings_fixture.yml | 1 - hathor_tests/others/test_cli_builder.py | 23 +- hathor_tests/others/test_hathor_settings.py | 1 - hathor_tests/p2p/test_bootstrap.py | 4 +- hathor_tests/p2p/test_capabilities.py | 42 +- hathor_tests/p2p/test_protocol.py | 12 +- hathor_tests/p2p/test_whitelist.py | 1085 ++++++++++++++++- hathor_tests/resources/p2p/test_status.py | 30 +- hathor_tests/unittest.py | 4 + hathor_tests/utils.py | 4 +- hathorlib/hathorlib/conf/mainnet.yml | 1 - 34 files changed, 1965 insertions(+), 229 deletions(-) create mode 100644 hathor/p2p/whitelist/__init__.py create mode 100644 hathor/p2p/whitelist/factory.py create mode 100644 hathor/p2p/whitelist/file_whitelist.py create mode 100644 hathor/p2p/whitelist/parsing.py create mode 100644 hathor/p2p/whitelist/peers_whitelist.py create mode 100644 hathor/p2p/whitelist/url_whitelist.py diff --git a/.gitignore b/.gitignore index a90f1a9ee..d0d3e7e80 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ keys.json # Pycharm .idea + +# Vscode +.vscode/ \ No newline at end of file diff --git a/hathor/builder/builder.py b/hathor/builder/builder.py index e15e8c914..611773cc3 100644 --- a/hathor/builder/builder.py +++ b/hathor/builder/builder.py @@ -41,6 +41,7 @@ from hathor.nanocontracts.sorter.types import NCSorterCallable from hathor.p2p.manager import ConnectionsManager from hathor.p2p.peer import PrivatePeer +from hathor.p2p.whitelist import PeersWhitelist from hathor.pubsub import PubSubManager from hathor.reactor import ReactorProtocol as Reactor from hathor.storage import RocksDBStorage @@ -191,6 +192,7 @@ def __init__(self) -> None: self._enable_ipv6: bool = False self._disable_ipv4: bool = False + self._peers_whitelist: PeersWhitelist | None = None self._nc_anti_mev: bool = True self._nc_storage_factory: NCStorageFactory | None = None @@ -349,6 +351,22 @@ def set_peer(self, peer: PrivatePeer) -> 'Builder': self._peer = peer return self + def set_url_whitelist(self, reactor: Reactor, url: str) -> 'Builder': + """Sets the peers whitelist to a URLPeersWhitelist. + + Args: + reactor: The Twisted reactor + url: The URL to fetch the whitelist from (required) + """ + self.check_if_can_modify() + if not url: + raise ValueError('url is required for set_url_whitelist') + from hathor.p2p.whitelist import URLPeersWhitelist + url_peers_whitelist = URLPeersWhitelist(reactor, url, False) + # We do not start the URLPeersWhitelist here, as it is started by the ConnectionsManager + self._peers_whitelist = url_peers_whitelist + return self + def _get_or_create_settings(self) -> HathorSettingsType: """Return the HathorSettings instance set on this builder, or a new one if not set.""" if self._settings is None: @@ -474,7 +492,7 @@ def _get_or_create_p2p_manager(self) -> ConnectionsManager: my_peer=my_peer, pubsub=self._get_or_create_pubsub(), ssl=enable_ssl, - whitelist_only=False, + peers_whitelist=self._peers_whitelist, rng=self._rng, enable_ipv6=self._enable_ipv6, disable_ipv4=self._disable_ipv4, @@ -779,6 +797,11 @@ def enable_event_queue(self) -> 'Builder': self._enable_event_queue = True return self + def set_whitelist(self, peers_whitelist: PeersWhitelist | None) -> 'Builder': + self.check_if_can_modify() + self._peers_whitelist = peers_whitelist + return self + def set_tx_storage(self, tx_storage: TransactionStorage) -> 'Builder': self.check_if_can_modify() self._tx_storage = tx_storage diff --git a/hathor/conf/mainnet.py b/hathor/conf/mainnet.py index e986304f6..b253d9dd8 100644 --- a/hathor/conf/mainnet.py +++ b/hathor/conf/mainnet.py @@ -24,7 +24,6 @@ MULTISIG_VERSION_BYTE=b'\x64', NETWORK_NAME='mainnet', BOOTSTRAP_DNS=['mainnet.hathor.network'], - ENABLE_PEER_WHITELIST=True, WHITELIST_URL='https://hathor-public-files.s3.amazonaws.com/whitelist_peer_ids', # Genesis stuff # output addr: HJB2yxxsHtudGGy3jmVeadwMfRi2zNCKKD diff --git a/hathor/manager.py b/hathor/manager.py index 99c5c18cc..639b42c7f 100644 --- a/hathor/manager.py +++ b/hathor/manager.py @@ -50,7 +50,6 @@ from hathor.nanocontracts.storage import NCBlockStorage, NCContractStorage from hathor.p2p.manager import ConnectionsManager from hathor.p2p.peer import PrivatePeer -from hathor.p2p.peer_id import PeerId from hathor.pubsub import EventArguments, HathorEvents, PubSubManager from hathor.reactor import ReactorProtocol as Reactor from hathor.reward_lock import is_spent_reward_locked @@ -233,9 +232,6 @@ def __init__( # Thread pool used to resolve pow when sending tokens self.pow_thread_pool = ThreadPool(minthreads=0, maxthreads=settings.MAX_POW_THREADS, name='Pow thread pool') - # List of whitelisted peers - self.peers_whitelist: list[PeerId] = [] - # List of capabilities of the peer if capabilities is not None: self.capabilities = capabilities @@ -887,24 +883,6 @@ def on_new_tx( def has_sync_version_capability(self) -> bool: return self._settings.CAPABILITY_SYNC_VERSION in self.capabilities - def add_peer_to_whitelist(self, peer_id: PeerId) -> None: - if not self._settings.ENABLE_PEER_WHITELIST: - return - - if peer_id in self.peers_whitelist: - self.log.info('peer already in whitelist', peer_id=peer_id) - else: - self.peers_whitelist.append(peer_id) - - def remove_peer_from_whitelist_and_disconnect(self, peer_id: PeerId) -> None: - if not self._settings.ENABLE_PEER_WHITELIST: - return - - if peer_id in self.peers_whitelist: - self.peers_whitelist.remove(peer_id) - # disconnect from node - self.connections.drop_connection_by_peer_id(peer_id) - def has_recent_activity(self) -> bool: current_timestamp = time.time() latest_blockchain_timestamp = self.tx_storage.latest_timestamp diff --git a/hathor/p2p/manager.py b/hathor/p2p/manager.py index 44dacdcf0..f147b276f 100644 --- a/hathor/p2p/manager.py +++ b/hathor/p2p/manager.py @@ -37,7 +37,7 @@ from hathor.p2p.states.ready import ReadyState from hathor.p2p.sync_factory import SyncAgentFactory from hathor.p2p.sync_version import SyncVersion -from hathor.p2p.utils import parse_whitelist +from hathor.p2p.whitelist import PeersWhitelist from hathor.pubsub import HathorEvents, PubSubManager from hathor.reactor import ReactorProtocol as Reactor from hathor.transaction import BaseTransaction @@ -48,9 +48,6 @@ logger = get_logger() -# The timeout in seconds for the whitelist GET request -WHITELIST_REQUEST_TIMEOUT = 45 - class _SyncRotateInfo(NamedTuple): candidates: list[PeerId] @@ -85,10 +82,10 @@ class GlobalRateLimiter: new_connection_from_queue: deque[PeerId] connecting_peers: dict[IStreamClientEndpoint, _ConnectingPeer] handshaking_peers: set[HathorProtocol] - whitelist_only: bool verified_peer_storage: VerifiedPeerStorage _sync_factories: dict[SyncVersion, SyncAgentFactory] _enabled_sync_versions: set[SyncVersion] + peers_whitelist: Optional[PeersWhitelist] rate_limiter: RateLimiter @@ -100,7 +97,7 @@ def __init__( pubsub: PubSubManager, ssl: bool, rng: Random, - whitelist_only: bool, + peers_whitelist: PeersWhitelist | None, enable_ipv6: bool, disable_ipv4: bool, ) -> None: @@ -187,17 +184,12 @@ def __init__( self.lc_connect.clock = self.reactor self.lc_connect_interval = 0.2 # seconds - # A timer to try to reconnect to the disconnect known peers. - if self._settings.ENABLE_PEER_WHITELIST: - self.wl_reconnect = LoopingCall(self.update_whitelist) - self.wl_reconnect.clock = self.reactor + # Whitelisted peers. + self.peers_whitelist: PeersWhitelist | None = peers_whitelist # Pubsub object to publish events self.pubsub = pubsub - # Parameter to explicitly enable whitelist-only mode, when False it will still check the whitelist for sync-v1 - self.whitelist_only = whitelist_only - # Parameter to enable IPv6 connections self.enable_ipv6 = enable_ipv6 @@ -273,7 +265,13 @@ def do_discovery(self) -> None: Do a discovery and connect on all discovery strategies. """ for peer_discovery in self.peer_discoveries: - coro = peer_discovery.discover_and_connect(self.connect_to_endpoint) + # Wrap connect_to_endpoint to register bootstrap peer IDs + def connect_with_bootstrap_registration(entrypoint: PeerEndpoint) -> None: + if self.peers_whitelist and entrypoint.peer_id is not None: + self.peers_whitelist.add_bootstrap_peer(entrypoint.peer_id) + self.connect_to_endpoint(entrypoint) + + coro = peer_discovery.discover_and_connect(connect_with_bootstrap_registration) Deferred.fromCoroutine(coro) def disable_rate_limiter(self) -> None: @@ -298,29 +296,14 @@ def start(self) -> None: self.lc_reconnect.start(5, now=False) self.lc_sync_update.start(self.lc_sync_update_interval, now=False) - if self._settings.ENABLE_PEER_WHITELIST: - self._start_whitelist_reconnect() + if self.peers_whitelist: + self.peers_whitelist.start(self.drop_connection_by_peer_id) for description in self.listen_address_descriptions: self.listen(description) self.do_discovery() - def _start_whitelist_reconnect(self) -> None: - # The deferred returned by the LoopingCall start method - # executes when the looping call stops running - # https://docs.twistedmatrix.com/en/stable/api/twisted.internet.task.LoopingCall.html - d = self.wl_reconnect.start(30) - d.addErrback(self._handle_whitelist_reconnect_err) - - def _handle_whitelist_reconnect_err(self, *args: Any, **kwargs: Any) -> None: - """ This method will be called when an exception happens inside the whitelist update - and ends up stopping the looping call. - We log the error and start the looping call again. - """ - self.log.error('whitelist reconnect had an exception. Start looping call again.', args=args, kwargs=kwargs) - self.reactor.callLater(30, self._start_whitelist_reconnect) - def _start_peer_connect_loop(self) -> None: # The deferred returned by the LoopingCall start method # executes when the looping call stops running @@ -349,6 +332,9 @@ def stop(self) -> None: if self.lc_sync_update.running: self.lc_sync_update.stop() + if self.peers_whitelist: + self.peers_whitelist.stop() + def _get_peers_count(self) -> PeerConnectionsMetrics: """Get a dict containing the count of peers in each state""" @@ -416,9 +402,9 @@ def on_peer_connect(self, protocol: HathorProtocol) -> None: self.log.warn('reached maximum number of connections', max_connections=self.max_connections) protocol.disconnect(force=True) return + self.connections.add(protocol) self.handshaking_peers.add(protocol) - self.pubsub.publish( HathorEvents.NETWORK_PEER_CONNECTED, protocol=protocol, @@ -597,47 +583,6 @@ def reconnect_to_all(self) -> None: for peer in list(self.verified_peer_storage.values()): self.connect_to_peer(peer, int(now)) - def update_whitelist(self) -> Deferred[None]: - from twisted.web.client import readBody - from twisted.web.http_headers import Headers - assert self._settings.WHITELIST_URL is not None - self.log.info('update whitelist') - d = self._http_agent.request( - b'GET', - self._settings.WHITELIST_URL.encode(), - Headers({'User-Agent': ['hathor-core']}), - None) - d.addCallback(readBody) - d.addTimeout(WHITELIST_REQUEST_TIMEOUT, self.reactor) - d.addCallback(self._update_whitelist_cb) - d.addErrback(self._update_whitelist_err) - - return d - - def _update_whitelist_err(self, *args: Any, **kwargs: Any) -> None: - self.log.error('update whitelist failed', args=args, kwargs=kwargs) - - def _update_whitelist_cb(self, body: bytes) -> None: - assert self.manager is not None - self.log.info('update whitelist got response') - try: - text = body.decode() - new_whitelist = parse_whitelist(text) - except Exception: - self.log.exception('failed to parse whitelist') - return - current_whitelist = set(self.manager.peers_whitelist) - peers_to_add = new_whitelist - current_whitelist - if peers_to_add: - self.log.info('add new peers to whitelist', peers=peers_to_add) - peers_to_remove = current_whitelist - new_whitelist - if peers_to_remove: - self.log.info('remove peers peers from whitelist', peers=peers_to_remove) - for peer_id in peers_to_add: - self.manager.add_peer_to_whitelist(peer_id) - for peer_id in peers_to_remove: - self.manager.remove_peer_from_whitelist_and_disconnect(peer_id) - def connect_to_peer(self, peer: UnverifiedPeer | PublicPeer, now: int) -> None: """ Attempts to connect if it is not connected to the peer. """ @@ -935,3 +880,25 @@ def reload_entrypoints_and_connections(self) -> None: self.log.warn('Killing all connections and resetting entrypoints...') self.disconnect_all_peers(force=True) self.my_peer.reload_entrypoints_from_source_file() + + def set_peers_whitelist(self, whitelist: PeersWhitelist | None) -> None: + """Replace the active whitelist at runtime.""" + if self.peers_whitelist is not None: + self.peers_whitelist.stop() + self.peers_whitelist = whitelist + if whitelist is not None: + whitelist.start(self.drop_connection_by_peer_id) + self._disconnect_non_whitelisted_peers() + + def _disconnect_non_whitelisted_peers(self) -> None: + """Disconnect all connected peers that are not in the current whitelist.""" + if not self.peers_whitelist: + return + self.log.info('Whitelist ON: disconnecting non-whitelisted peers...') + connections_snapshot = list(self.connections) + for conn in connections_snapshot: + peer_id = conn.get_peer_id() + if peer_id is None: + continue + if not self.peers_whitelist.is_peer_whitelisted(peer_id): + conn.disconnect(reason='Whitelist updated', force=True) diff --git a/hathor/p2p/resources/status.py b/hathor/p2p/resources/status.py index 12dcf2e79..7e7e4fea8 100644 --- a/hathor/p2p/resources/status.py +++ b/hathor/p2p/resources/status.py @@ -95,6 +95,11 @@ def render_GET(self, request): best_blockchain = to_serializable_best_blockchain(raw_best_blockchain) best_block_tips = [{'hash': best_block.hash_hex, 'height': best_block.static_metadata.height}] + whitelist = self.manager.connections.peers_whitelist + if whitelist is not None: + peer_ids = whitelist.current_whitelist() + else: + peer_ids = set() data = { 'server': { 'id': str(self.manager.connections.my_peer.id), @@ -104,7 +109,7 @@ def render_GET(self, request): 'uptime': now - self.manager.start_time, 'entrypoints': self.manager.connections.my_peer.info.entrypoints_as_str(), }, - 'peers_whitelist': [str(peer_id) for peer_id in self.manager.peers_whitelist], + 'peers_whitelist': [str(peer_id) for peer_id in peer_ids], 'known_peers': known_peers, 'connections': { 'connected_peers': connected_peers, diff --git a/hathor/p2p/states/hello.py b/hathor/p2p/states/hello.py index 47c9cf4e5..06ca2a108 100644 --- a/hathor/p2p/states/hello.py +++ b/hathor/p2p/states/hello.py @@ -102,7 +102,7 @@ def handle_hello(self, payload: str) -> None: protocol.send_error_and_close_connection('Invalid payload.') return - if self._settings.ENABLE_PEER_WHITELIST and self._settings.CAPABILITY_WHITELIST not in data['capabilities']: + if self._settings.CAPABILITY_WHITELIST not in data['capabilities']: # If peer is not sending whitelist capability we must close the connection protocol.send_error_and_close_connection('Must have whitelist capability.') return @@ -184,4 +184,4 @@ def _parse_sync_versions(hello_data: dict[str, Any]) -> set[SyncVersion]: return set(SyncVersion(x) for x in recognized_values) else: # XXX: implied value when sync-version capability isn't present - return {SyncVersion.V1_1} + return {SyncVersion.V2} diff --git a/hathor/p2p/states/peer_id.py b/hathor/p2p/states/peer_id.py index cf7e0fd44..66edde341 100644 --- a/hathor/p2p/states/peer_id.py +++ b/hathor/p2p/states/peer_id.py @@ -116,7 +116,7 @@ async def handle_peer_id(self, payload: str) -> None: return # is it on the whitelist? - if peer.id and self._should_block_peer(peer.id): + if not self._is_peer_allowed(peer.id): if self._settings.WHITELIST_WARN_BLOCKED_PEERS: protocol.send_error_and_close_connection(f'Blocked (by {peer.id}). Get in touch with Hathor team.') else: @@ -161,30 +161,9 @@ async def handle_peer_id(self, payload: str) -> None: self.send_ready() - def _should_block_peer(self, peer_id: PeerId) -> bool: - """ Determine if peer should not be allowed to connect. - - Currently this is only because the peer is not in a whitelist and whitelist blocking is active. - """ - peer_is_whitelisted = peer_id in self.protocol.node.peers_whitelist - # never block whitelisted peers - if peer_is_whitelisted: - return False - - # when ENABLE_PEER_WHITELIST is set, we check if we're on sync-v1 to block non-whitelisted peers - if self._settings.ENABLE_PEER_WHITELIST: - assert self.protocol.sync_version is not None - if not peer_is_whitelisted: - if self.protocol.sync_version.is_v1(): - return True - elif self._settings.USE_PEER_WHITELIST_ON_SYNC_V2: - return True - - # otherwise we block non-whitelisted peers when on "whitelist-only mode" - if self.protocol.connections is not None: - protocol_is_whitelist_only = self.protocol.connections.whitelist_only - if protocol_is_whitelist_only and not peer_is_whitelisted: - return True - - # default is not blocking, this will be sync-v2 peers not on whitelist when not on whitelist-only mode - return False + def _is_peer_allowed(self, peer_id: PeerId) -> bool: + """Return True if peer is allowed to connect; False otherwise.""" + peers_whitelist = self.protocol.connections.peers_whitelist + if peers_whitelist is None: + return True + return peers_whitelist.is_peer_whitelisted(peer_id) diff --git a/hathor/p2p/sync_version.py b/hathor/p2p/sync_version.py index 6de1538d2..498ba0eda 100644 --- a/hathor/p2p/sync_version.py +++ b/hathor/p2p/sync_version.py @@ -22,7 +22,6 @@ class SyncVersion(Enum): # to no match different values and in turn not select a certain protocol, this can be done intentionally, for # example, peers using `v2-fake` (which just uses sync-v1) will not connect to peers using `v2-alpha`, and so # on. - V1_1 = 'v1.1' V2 = 'v2' def __str__(self): @@ -36,17 +35,11 @@ def get_priority(self) -> int: # XXX: these values are only used internally and in memory, there is no need to keep them consistency, for # example, if we need more granularity, we can just add a 0 to all values and use the values in between, # although this shouldn't really be necessary - if self == SyncVersion.V1_1: - return 11 - elif self == SyncVersion.V2: + if self == SyncVersion.V2: return 20 else: raise ValueError('value is either invalid for this enum or not implemented') - def is_v1(self) -> bool: - """Return True for V1_1.""" - return self.get_priority() < 20 - # XXX: total_ordering decorator will implement the other methods: __le__, __gt__, and __ge__ def __lt__(self, other): """Used to sort versions by considering the value on get_priority.""" diff --git a/hathor/p2p/utils.py b/hathor/p2p/utils.py index 50b7718c7..d67bd1d83 100644 --- a/hathor/p2p/utils.py +++ b/hathor/p2p/utils.py @@ -24,6 +24,7 @@ from cryptography.hazmat.primitives.serialization import load_pem_private_key from cryptography.x509 import Certificate from cryptography.x509.oid import NameOID +from structlog import get_logger from twisted.internet.interfaces import IAddress from hathor.conf.get_settings import get_global_settings @@ -31,9 +32,10 @@ from hathor.indexes.height_index import HeightInfo from hathor.p2p.peer_discovery import DNSPeerDiscovery from hathor.p2p.peer_endpoint import PeerEndpoint -from hathor.p2p.peer_id import PeerId from hathor.transaction.genesis import get_representation_for_all_genesis +logger = get_logger() + def discover_hostname(timeout: float | None = None) -> Optional[str]: """ Try to discover your hostname. It is a synchronous operation and @@ -143,28 +145,6 @@ def parse_file(text: str, *, header: Optional[str] = None) -> list[str]: return list(nonblank_lines) -def parse_whitelist(text: str, *, header: Optional[str] = None) -> set[PeerId]: - """ Parses the list of whitelist peer ids - - Example: - - parse_whitelist('''hathor-whitelist -# node1 - 2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 - -2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 - -# node3 -G2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 -2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 -''') - {'2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'} - - """ - lines = parse_file(text, header=header) - return {PeerId(line.split()[0]) for line in lines} - - def format_address(addr: IAddress) -> str: """ Return a string with '{host}:{port}' when possible, otherwise use the addr's __str__ """ diff --git a/hathor/p2p/whitelist/__init__.py b/hathor/p2p/whitelist/__init__.py new file mode 100644 index 000000000..61ab5a76f --- /dev/null +++ b/hathor/p2p/whitelist/__init__.py @@ -0,0 +1,50 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .factory import ( + WHITELIST_SPEC_DEFAULT, + WHITELIST_SPEC_DISABLED, + WHITELIST_SPEC_HATHORLABS, + WHITELIST_SPEC_NONE, + create_peers_whitelist, +) +from .file_whitelist import FilePeersWhitelist +from .parsing import WhitelistPolicy, parse_whitelist, parse_whitelist_with_policy +from .peers_whitelist import ( + WHITELIST_REFRESH_INTERVAL, + WHITELIST_RETRY_INTERVAL_MAX, + WHITELIST_RETRY_INTERVAL_MIN, + OnRemoveCallbackType, + PeersWhitelist, +) +from .url_whitelist import WHITELIST_REQUEST_TIMEOUT, URLPeersWhitelist + +__all__ = [ + 'WhitelistPolicy', + 'parse_whitelist', + 'parse_whitelist_with_policy', + 'PeersWhitelist', + 'OnRemoveCallbackType', + 'WHITELIST_REFRESH_INTERVAL', + 'WHITELIST_RETRY_INTERVAL_MIN', + 'WHITELIST_RETRY_INTERVAL_MAX', + 'FilePeersWhitelist', + 'URLPeersWhitelist', + 'WHITELIST_REQUEST_TIMEOUT', + 'create_peers_whitelist', + 'WHITELIST_SPEC_DEFAULT', + 'WHITELIST_SPEC_HATHORLABS', + 'WHITELIST_SPEC_NONE', + 'WHITELIST_SPEC_DISABLED', +] diff --git a/hathor/p2p/whitelist/factory.py b/hathor/p2p/whitelist/factory.py new file mode 100644 index 000000000..5e9ee9825 --- /dev/null +++ b/hathor/p2p/whitelist/factory.py @@ -0,0 +1,62 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import TYPE_CHECKING + +from hathor.p2p.whitelist.file_whitelist import FilePeersWhitelist +from hathor.p2p.whitelist.peers_whitelist import PeersWhitelist +from hathor.p2p.whitelist.url_whitelist import URLPeersWhitelist +from hathor.reactor import ReactorProtocol as Reactor + +if TYPE_CHECKING: + from hathor.conf.settings import HathorSettings + +# Whitelist specification constants +WHITELIST_SPEC_DEFAULT = 'default' +WHITELIST_SPEC_HATHORLABS = 'hathorlabs' +WHITELIST_SPEC_NONE = 'none' +WHITELIST_SPEC_DISABLED = 'disabled' + + +def create_peers_whitelist( + reactor: Reactor, + whitelist_spec: str, + settings: 'HathorSettings', +) -> 'PeersWhitelist | None': + """Factory function to create PeersWhitelist from a specification string. + + Args: + reactor: The Twisted reactor + whitelist_spec: Whitelist specification - can be 'default', 'hathorlabs', 'none', 'disabled', + a file path, or a URL + settings: Hathor settings containing WHITELIST_URL + + Returns: + PeersWhitelist instance or None if disabled + """ + peers_whitelist: PeersWhitelist | None = None + spec_lower = whitelist_spec.lower() + + if spec_lower in (WHITELIST_SPEC_DEFAULT, WHITELIST_SPEC_HATHORLABS): + peers_whitelist = URLPeersWhitelist(reactor, str(settings.WHITELIST_URL), True) + elif spec_lower in (WHITELIST_SPEC_NONE, WHITELIST_SPEC_DISABLED): + peers_whitelist = None + elif os.path.isfile(whitelist_spec): + peers_whitelist = FilePeersWhitelist(reactor, whitelist_spec) + else: + # URLPeersWhitelist class rejects non-url paths. + peers_whitelist = URLPeersWhitelist(reactor, whitelist_spec, True) + + return peers_whitelist diff --git a/hathor/p2p/whitelist/file_whitelist.py b/hathor/p2p/whitelist/file_whitelist.py new file mode 100644 index 000000000..6fa880333 --- /dev/null +++ b/hathor/p2p/whitelist/file_whitelist.py @@ -0,0 +1,80 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, cast + +from twisted.internet import threads +from twisted.internet.defer import Deferred + +from hathor.p2p.whitelist.parsing import parse_whitelist_with_policy +from hathor.p2p.whitelist.peers_whitelist import PeersWhitelist +from hathor.reactor import ReactorProtocol as Reactor + + +class FilePeersWhitelist(PeersWhitelist): + def __init__(self, reactor: Reactor, path: str) -> None: + super().__init__(reactor) + self._path = path + + def path(self) -> str: + return self._path + + def source(self) -> str | None: + """Return the file path as the whitelist source.""" + return self._path + + def refresh(self) -> Deferred[None]: + return self._unsafe_update() + + def _read_file(self) -> str: + """Read the whitelist file. Runs in a thread to avoid blocking.""" + with open(self._path, 'r', encoding='utf-8') as fp: + return fp.read() + + def _process_content(self, content: str) -> None: + """Process the whitelist file content after reading.""" + try: + new_whitelist, new_policy = parse_whitelist_with_policy(content) + except ValueError as e: + self.log.error('Failed to parse whitelist file content', path=self._path, error=str(e)) + self._on_update_failure() + return + except Exception: + self.log.exception('Unexpected error parsing whitelist file', path=self._path) + self._on_update_failure() + return + + self._apply_whitelist_update(new_whitelist, new_policy) + + def _handle_read_error(self, failure: Any) -> None: + """Handle errors when reading the whitelist file.""" + self._on_update_failure() + error = failure.value + if isinstance(error, FileNotFoundError): + self.log.warning('Whitelist file not found, keeping existing whitelist', path=self._path) + elif isinstance(error, PermissionError): + self.log.warning('Permission denied reading whitelist file, keeping existing whitelist', path=self._path) + else: + self.log.error('Failed to read whitelist file', path=self._path, error=str(error)) + + def _unsafe_update(self) -> Deferred[None]: + """ + Implementation of base class function. + Reads the file in the class path using a thread to avoid blocking. + """ + d: Deferred[str] = threads.deferToThread(self._read_file) + d.addCallback(self._process_content) + d.addErrback(self._handle_read_error) + # Cast to Deferred[None] since callbacks transform the result + return cast(Deferred[None], d) diff --git a/hathor/p2p/whitelist/parsing.py b/hathor/p2p/whitelist/parsing.py new file mode 100644 index 000000000..eb62c08bc --- /dev/null +++ b/hathor/p2p/whitelist/parsing.py @@ -0,0 +1,118 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from typing import Optional + +from structlog import get_logger + +from hathor.p2p.peer_id import PeerId +from hathor.p2p.utils import parse_file + +logger = get_logger() + + +class WhitelistPolicy(Enum): + """Policy types for whitelist behavior.""" + ALLOW_ALL = 'allow-all' + ONLY_WHITELISTED_PEERS = 'only-whitelisted-peers' + + +def parse_whitelist(text: str, *, header: Optional[str] = None) -> set[PeerId]: + """ Parses the list of whitelist peer ids + + Example: + + parse_whitelist('''hathor-whitelist +# node1 + 2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 + +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 + +# node3 +G2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +''') + {'2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'} + + """ + lines = parse_file(text, header=header) + return {PeerId(line.split()[0]) for line in lines} + + +def parse_whitelist_with_policy( + text: str, + *, + header: Optional[str] = None +) -> tuple[set[PeerId], WhitelistPolicy]: + """Parses the whitelist file and extracts both peer IDs and policy. + + The policy line (optional) must appear in the header, before any peer IDs. + Format: policy: + + Policy types: + - allow-all: Allow connections from any peer + - only-whitelisted-peers: Only allow connections from listed peers (default) + + Example: + + parse_whitelist_with_policy('''hathor-whitelist +policy: allow-all +''') + (set(), WhitelistPolicy.ALLOW_ALL) + + parse_whitelist_with_policy('''hathor-whitelist +# This whitelist only allows specific peers +policy: only-whitelisted-peers +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +''') + ({'2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'}, WhitelistPolicy.ONLY_WHITELISTED_PEERS) + + parse_whitelist_with_policy('''hathor-whitelist +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +''') + ({'2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'}, WhitelistPolicy.ONLY_WHITELISTED_PEERS) + """ + lines = parse_file(text, header=header) + policy = WhitelistPolicy.ONLY_WHITELISTED_PEERS # default + + peer_lines: list[str] = [] + + for line in lines: + if line.startswith('policy:'): + if len(peer_lines) > 0: + raise ValueError('policy must be defined in the header, before any peer IDs') + policy_value = line.split(':', 1)[1].strip().lower() + try: + policy = WhitelistPolicy(policy_value) + except ValueError: + raise ValueError(f'invalid whitelist policy: {policy_value}') + else: + peer_lines.append(line) + + peers = {p for line in peer_lines if (p := _safe_parse_peer_id(line)) is not None} + return peers, policy + + +def _safe_parse_peer_id(line: str) -> PeerId | None: + """Safely parse a peer ID from a whitelist line. + + Returns the PeerId if valid, or None if parsing fails. + Logs a warning for invalid peer IDs. + """ + try: + return PeerId(line.split()[0]) + except (ValueError, IndexError) as e: + logger.warning('Invalid peer ID in whitelist', line=line, error=str(e)) + return None diff --git a/hathor/p2p/whitelist/peers_whitelist.py b/hathor/p2p/whitelist/peers_whitelist.py new file mode 100644 index 000000000..197ffcdbf --- /dev/null +++ b/hathor/p2p/whitelist/peers_whitelist.py @@ -0,0 +1,177 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Any, Callable + +from structlog import get_logger +from twisted.internet.defer import Deferred +from twisted.internet.task import LoopingCall + +from hathor.p2p.peer_id import PeerId +from hathor.p2p.whitelist.parsing import WhitelistPolicy +from hathor.reactor import ReactorProtocol as Reactor + +logger = get_logger() + +WHITELIST_REFRESH_INTERVAL = 30 + +# Exponential backoff constants for retry intervals +WHITELIST_RETRY_INTERVAL_MIN = 30 +WHITELIST_RETRY_INTERVAL_MAX = 300 + +OnRemoveCallbackType = Callable[[PeerId], None] | None + + +class PeersWhitelist(ABC): + def __init__(self, reactor: Reactor) -> None: + self.log = logger.new() + self._reactor = reactor + self.lc_refresh = LoopingCall(self.update) + self.lc_refresh.clock = self._reactor + self._current: set[PeerId] = set() + self._policy: WhitelistPolicy = WhitelistPolicy.ONLY_WHITELISTED_PEERS + self._on_remove_callback: OnRemoveCallbackType = None + self._is_running: bool = False + self._consecutive_failures: int = 0 + self._has_successful_fetch: bool = False + self._bootstrap_peers: set[PeerId] = set() + + def add_bootstrap_peer(self, peer_id: PeerId) -> None: + """Add a bootstrap peer ID. These are allowed during grace period.""" + self._bootstrap_peers.add(peer_id) + self.log.debug('Bootstrap peer added', peer_id=peer_id) + + def start(self, on_remove_callback: OnRemoveCallbackType) -> None: + self._on_remove_callback = on_remove_callback + self._start_lc() + + def _start_lc(self) -> None: + # The deferred returned by the LoopingCall start method executes when the looping call stops running. + # https://docs.twistedmatrix.com/en/stable/api/twisted.internet.task.LoopingCall.html + d = self.lc_refresh.start(WHITELIST_REFRESH_INTERVAL) + d.addErrback(self._handle_refresh_err) + + def stop(self) -> None: + if self.lc_refresh.running: + self.lc_refresh.stop() + + def _get_retry_interval(self) -> int: + """Calculate retry interval with exponential backoff. + + Returns interval in seconds: 30 → 60 → 120 → max 300. + """ + interval = WHITELIST_RETRY_INTERVAL_MIN * (2 ** self._consecutive_failures) + return min(interval, WHITELIST_RETRY_INTERVAL_MAX) + + def _on_update_success(self) -> None: + """Called when whitelist update succeeds. Resets backoff counter.""" + self._consecutive_failures = 0 + + def _on_update_failure(self) -> None: + """Called when whitelist update fails. Increments backoff counter.""" + self._consecutive_failures += 1 + + def _handle_refresh_err(self, *args: Any, **kwargs: Any) -> None: + """This method will be called when an exception happens inside the whitelist update + and ends up stopping the looping call. + We log the error and start the looping call again with exponential backoff. + """ + self._on_update_failure() + retry_interval = self._get_retry_interval() + self.log.error( + 'whitelist refresh had an exception. Start looping call again.', + args=args, + kwargs=kwargs, + retry_interval=retry_interval, + consecutive_failures=self._consecutive_failures + ) + self._reactor.callLater(retry_interval, self._start_lc) + + def update(self) -> Deferred[None]: + # Avoiding re-entrancy. If running, should not update once more. + if self._is_running: + self.log.warning('whitelist update already running, skipping execution.') + d: Deferred[None] = Deferred() + d.callback(None) + return d + + self._is_running = True + d = self._unsafe_update() + d.addBoth(lambda _: setattr(self, '_is_running', False)) + return d + + def add_peer(self, peer_id: PeerId) -> None: + """ Adds a peer to the current whitelist. """ + if peer_id not in self._current: + self._current.add(peer_id) + self.log.info('Peer added to whitelist', peer_id=peer_id) + + def current_whitelist(self) -> set[PeerId]: + """ Returns the current whitelist as a set of PeerId.""" + return self._current + + def policy(self) -> WhitelistPolicy: + """ Returns the current whitelist policy.""" + return self._policy + + def is_peer_whitelisted(self, peer_id: PeerId) -> bool: + """ Returns True if peer is whitelisted or policy is ALLOW_ALL. + + During the grace period (before first successful fetch), only bootstrap peers + are allowed to prevent connecting to arbitrary peers before the whitelist is loaded. + """ + # Grace period: only allow bootstrap peers until first successful fetch + if not self._has_successful_fetch: + return peer_id in self._bootstrap_peers + if self._policy == WhitelistPolicy.ALLOW_ALL: + return True + return peer_id in self._current + + def _log_diff(self, current_whitelist: set[PeerId], new_whitelist: set[PeerId]) -> None: + peers_to_add = new_whitelist - current_whitelist + if peers_to_add: + self.log.info('add new peers to whitelist', peers=peers_to_add) + + peers_to_remove = current_whitelist - new_whitelist + if peers_to_remove: + self.log.info('remove peers from whitelist', peers=peers_to_remove) + + def _apply_whitelist_update(self, new_whitelist: set[PeerId], new_policy: WhitelistPolicy) -> None: + """Apply a whitelist update: log diff, call remove callbacks, and update state. + + This is the common logic used by both URL and file-based whitelists after + successfully parsing new whitelist content. + """ + current_whitelist = set(self._current) + self._log_diff(current_whitelist, new_whitelist) + + peers_to_remove = current_whitelist - new_whitelist + for peer_id in peers_to_remove: + if self._on_remove_callback: + self._on_remove_callback(peer_id) + + self._current = new_whitelist + self._policy = new_policy + self._has_successful_fetch = True + self._on_update_success() + + @abstractmethod + def source(self) -> str | None: + """Return the source of the whitelist (URL or file path).""" + raise NotImplementedError + + @abstractmethod + def _unsafe_update(self) -> Deferred[None]: + raise NotImplementedError diff --git a/hathor/p2p/whitelist/url_whitelist.py b/hathor/p2p/whitelist/url_whitelist.py new file mode 100644 index 000000000..85f04cb73 --- /dev/null +++ b/hathor/p2p/whitelist/url_whitelist.py @@ -0,0 +1,123 @@ +# Copyright 2021 Hathor Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any +from urllib.parse import urlparse + +from twisted.internet.defer import Deferred +from twisted.web.client import Agent + +from hathor.p2p.whitelist.parsing import parse_whitelist_with_policy +from hathor.p2p.whitelist.peers_whitelist import PeersWhitelist +from hathor.reactor import ReactorProtocol as Reactor + +# The timeout in seconds for the whitelist GET request +WHITELIST_REQUEST_TIMEOUT = 45 + + +class URLPeersWhitelist(PeersWhitelist): + def __init__(self, reactor: Reactor, url: str | None, mainnet: bool = False) -> None: + super().__init__(reactor) + self._url: str | None = url + self._http_agent = Agent(self._reactor) + + if self._url is None: + return + if self._url.lower() == 'none': + self._url = None + return + + result = urlparse(self._url) + if mainnet: + if result.scheme != 'https': + raise ValueError(f'invalid scheme: {self._url}') + + if not result.netloc: + raise ValueError(f'invalid url: {self._url}') + + def url(self) -> str | None: + return self._url + + def source(self) -> str | None: + """Return the URL as the whitelist source.""" + return self._url + + def _update_whitelist_err(self, failure: Any) -> None: + from twisted.internet.defer import TimeoutError + self._on_update_failure() + error = failure.value + if isinstance(error, TimeoutError): + self.log.warning( + 'Whitelist URL request timed out', + url=self._url, + timeout=WHITELIST_REQUEST_TIMEOUT, + consecutive_failures=self._consecutive_failures + ) + else: + self.log.error( + 'Failed to fetch whitelist from URL', + url=self._url, + error_type=type(error).__name__, + error=str(error), + consecutive_failures=self._consecutive_failures + ) + + def _update_whitelist_cb(self, body: bytes) -> None: + self.log.info('update whitelist got response') + try: + text = body.decode('utf-8') + except UnicodeDecodeError as e: + self.log.error('Failed to decode whitelist response', url=self._url, error=str(e)) + self._on_update_failure() + return + + try: + new_whitelist, new_policy = parse_whitelist_with_policy(text) + except ValueError as e: + self.log.error('Failed to parse whitelist content', url=self._url, error=str(e)) + self._on_update_failure() + return + except Exception: + self.log.exception('Unexpected error parsing whitelist', url=self._url) + self._on_update_failure() + return + + self._apply_whitelist_update(new_whitelist, new_policy) + + def _unsafe_update(self) -> Deferred[None]: + """ + Implementation of the child class of PeersWhitelist, called by update() + to fetch data from the provided url. + """ + from twisted.web.client import readBody + from twisted.web.http_headers import Headers + + # Guard against URL being None (e.g., when set to "none" string) + if self._url is None: + self.log.debug('skipping whitelist update, url is None') + d: Deferred[None] = Deferred() + d.callback(None) + return d + + self.log.info('update whitelist') + d = self._http_agent.request( + b'GET', + self._url.encode(), + Headers({'User-Agent': ['hathor-core']}), + None) + d.addCallback(readBody) # type: ignore[call-overload] + d.addTimeout(WHITELIST_REQUEST_TIMEOUT, self._reactor) + d.addCallback(self._update_whitelist_cb) # type: ignore[call-overload] + d.addErrback(self._update_whitelist_err) + return d diff --git a/hathor/sysctl/p2p/manager.py b/hathor/sysctl/p2p/manager.py index 9f9856a42..ab3b3a96c 100644 --- a/hathor/sysctl/p2p/manager.py +++ b/hathor/sysctl/p2p/manager.py @@ -13,14 +13,41 @@ # limitations under the License. import os +from dataclasses import dataclass +from enum import Enum from hathor.p2p.manager import ConnectionsManager from hathor.p2p.peer_id import PeerId from hathor.p2p.sync_version import SyncVersion from hathor.p2p.utils import discover_hostname +from hathor.p2p.whitelist import PeersWhitelist, WhitelistPolicy from hathor.sysctl.exception import SysctlException from hathor.sysctl.sysctl import Sysctl, signal_handler_safe + +class WhitelistState(Enum): + """State of the whitelist.""" + DISABLED = 'disabled' # No whitelist configured + OFF = 'off' # Whitelist configured but not being followed + ON = 'on' # Whitelist configured and being followed + + +@dataclass(frozen=True) +class WhitelistStatus: + """Status information for the whitelist. + + Attributes: + state: Current state of the whitelist (disabled/off/on) + policy: The whitelist policy (only set when state is not DISABLED) + peer_count: Number of peers in the whitelist + source: The source URL or file path (only set when state is not DISABLED) + """ + state: WhitelistState + policy: WhitelistPolicy | None = None + peer_count: int = 0 + source: str | None = None + + AUTO_HOSTNAME_TIMEOUT_SECONDS: float = 5 @@ -39,8 +66,6 @@ def parse_text(text: str) -> list[str]: def parse_sync_version(name: str) -> SyncVersion: match name.strip(): - case 'v1': - return SyncVersion.V1_1 case 'v2': return SyncVersion.V2 case _: @@ -49,8 +74,6 @@ def parse_sync_version(name: str) -> SyncVersion: def pretty_sync_version(sync_version: SyncVersion) -> str: match sync_version: - case SyncVersion.V1_1: - return 'v1' case SyncVersion.V2: return 'v2' case _: @@ -62,6 +85,7 @@ def __init__(self, connections: ConnectionsManager) -> None: super().__init__() self.connections = connections + self._suspended_whitelist: PeersWhitelist | None = None self.register( 'max_enabled_sync', self.get_max_enabled_sync, @@ -122,6 +146,17 @@ def __init__(self, connections: ConnectionsManager) -> None: None, self.reload_entrypoints_and_connections, ) + self.register( + 'whitelist', + self.get_whitelist, + self.set_whitelist, + ) + + self.register( + 'whitelist.status', + self.whitelist_status, + None, + ) def set_force_sync_rotate(self) -> None: """Force a sync rotate.""" @@ -269,3 +304,66 @@ def refresh_auto_hostname(self) -> None: def reload_entrypoints_and_connections(self) -> None: """Kill all connections and reload entrypoints from the peer config file.""" self.connections.reload_entrypoints_and_connections() + + def get_whitelist(self) -> str: + """Get source of current whitelist (URL or path).""" + whitelist = self.connections.peers_whitelist + if whitelist is not None: + source = whitelist.source() + return source if source is not None else 'none' + return 'none' + + def set_whitelist(self, new_whitelist: str) -> None: + """Set the whitelist-only mode. If 'on' or 'off', simply changes the + following status of current whitelist. If an URL or Filepath, changes + the whitelist object, following it by default. + It does not support eliminating the whitelist (passing None).""" + + option: str = new_whitelist.lower().strip() + if option == 'on': + if self._suspended_whitelist is None: + return + self.connections.set_peers_whitelist(self._suspended_whitelist) + self._suspended_whitelist = None + return + if option == 'off': + if self.connections.peers_whitelist is None: + return + self._suspended_whitelist = self.connections.peers_whitelist + self.connections.set_peers_whitelist(None) + return + + from hathor.p2p.whitelist import create_peers_whitelist + whitelist = create_peers_whitelist( + self.connections.reactor, + new_whitelist, + self.connections._settings, + ) + + if whitelist is None: + raise SysctlException('Sysctl does not allow whitelist swap to None. Use "off" to disable it.') + + self._suspended_whitelist = None + self.connections.set_peers_whitelist(whitelist) + + def whitelist_status(self) -> WhitelistStatus: + """Return structured status information about the whitelist.""" + if self.connections.peers_whitelist is not None: + whitelist = self.connections.peers_whitelist + return WhitelistStatus( + state=WhitelistState.ON, + policy=whitelist.policy(), + peer_count=len(whitelist.current_whitelist()), + source=whitelist.source(), + ) + + if self._suspended_whitelist is not None: + whitelist = self._suspended_whitelist + return WhitelistStatus( + state=WhitelistState.OFF, + policy=whitelist.policy(), + peer_count=len(whitelist.current_whitelist()), + source=whitelist.source(), + ) + + return WhitelistStatus(state=WhitelistState.DISABLED) diff --git a/hathor_cli/builder.py b/hathor_cli/builder.py index 6cc886c78..7ffd9d002 100644 --- a/hathor_cli/builder.py +++ b/hathor_cli/builder.py @@ -20,6 +20,7 @@ from structlog import get_logger +from hathor.conf.settings import HathorSettings from hathor.transaction.storage.rocksdb_storage import CacheConfig from hathor_cli.run_node_args import RunNodeArgs from hathor_cli.side_dag import SideDagArgs @@ -39,6 +40,7 @@ from hathor.p2p.manager import ConnectionsManager from hathor.p2p.peer import PrivatePeer from hathor.p2p.peer_endpoint import PeerEndpoint +from hathor.p2p.whitelist import PeersWhitelist, create_peers_whitelist from hathor.p2p.utils import discover_hostname, get_genesis_short_hash from hathor.pubsub import PubSubManager from hathor.reactor import ReactorProtocol as Reactor @@ -297,13 +299,23 @@ def create_manager(self, reactor: Reactor) -> HathorManager: cpu_mining_service = CpuMiningService() + # Check whitelist pathing. Default values: + whitelist_spec = self._args.x_p2p_whitelist or 'default' + whitelist_spec = whitelist_spec.strip() + + peers_whitelist = create_peers_whitelist( + reactor=reactor, + whitelist_spec=whitelist_spec, + settings=settings, + ) + p2p_manager = ConnectionsManager( settings=settings, reactor=reactor, my_peer=peer, pubsub=pubsub, ssl=True, - whitelist_only=False, + peers_whitelist=peers_whitelist, rng=Random(), enable_ipv6=self._args.x_enable_ipv6, disable_ipv4=self._args.x_disable_ipv4, diff --git a/hathor_cli/run_node.py b/hathor_cli/run_node.py index 5b4a34a66..9d3c82d19 100644 --- a/hathor_cli/run_node.py +++ b/hathor_cli/run_node.py @@ -174,6 +174,11 @@ def create_parser(cls) -> ArgumentParser: help='Enables listening on IPv6 interface and connecting to IPv6 peers') parser.add_argument('--x-disable-ipv4', action='store_true', help='Disables connecting to IPv4 peers') + + parser.add_argument("--x-p2p-whitelist-only", action="store_true", + help="Node will only connect to peers on the whitelist") + parser.add_argument("--x-p2p-whitelist", help="Add whitelist to follow from since boot.") + possible_nc_exec_logs = [config.value for config in NCLogConfig] parser.add_argument('--nc-exec-logs', default=NCLogConfig.NONE, choices=possible_nc_exec_logs, help=f'Enable saving Nano Contracts execution logs. One of {possible_nc_exec_logs}') diff --git a/hathor_cli/run_node_args.py b/hathor_cli/run_node_args.py index 7fe1f2e2e..8486cbdf8 100644 --- a/hathor_cli/run_node_args.py +++ b/hathor_cli/run_node_args.py @@ -93,6 +93,8 @@ class RunNodeArgs(BaseModel): x_enable_ipv6: bool x_disable_ipv4: bool localnet: bool + x_p2p_whitelist_only: bool + x_p2p_whitelist: Optional[str] nc_indexes: bool nc_exec_logs: NCLogConfig nc_exec_fail_trace: bool diff --git a/hathor_tests/others/fixtures/invalid_byte_hathor_settings_fixture.yml b/hathor_tests/others/fixtures/invalid_byte_hathor_settings_fixture.yml index b05b8f246..32fe7253b 100644 --- a/hathor_tests/others/fixtures/invalid_byte_hathor_settings_fixture.yml +++ b/hathor_tests/others/fixtures/invalid_byte_hathor_settings_fixture.yml @@ -3,7 +3,7 @@ MULTISIG_VERSION_BYTE: 64 NETWORK_NAME: testing BOOTSTRAP_DNS: - mainnet.hathor.network -ENABLE_PEER_WHITELIST: true + WHITELIST_URL: https://hathor-public-files.s3.amazonaws.com/whitelist_peer_ids GENESIS_OUTPUT_SCRIPT: 76a9147fd4ae0e4fb2d2854e76d359029d8078bb99649e88ac diff --git a/hathor_tests/others/fixtures/invalid_features_hathor_settings_fixture.yml b/hathor_tests/others/fixtures/invalid_features_hathor_settings_fixture.yml index c2103afef..bda792d45 100644 --- a/hathor_tests/others/fixtures/invalid_features_hathor_settings_fixture.yml +++ b/hathor_tests/others/fixtures/invalid_features_hathor_settings_fixture.yml @@ -3,7 +3,7 @@ MULTISIG_VERSION_BYTE: '64' NETWORK_NAME: testing BOOTSTRAP_DNS: - mainnet.hathor.network -ENABLE_PEER_WHITELIST: true + WHITELIST_URL: https://hathor-public-files.s3.amazonaws.com/whitelist_peer_ids GENESIS_OUTPUT_SCRIPT: 76a9147fd4ae0e4fb2d2854e76d359029d8078bb99649e88ac diff --git a/hathor_tests/others/fixtures/missing_hathor_settings_fixture.yml b/hathor_tests/others/fixtures/missing_hathor_settings_fixture.yml index 81719c264..a883cf5db 100644 --- a/hathor_tests/others/fixtures/missing_hathor_settings_fixture.yml +++ b/hathor_tests/others/fixtures/missing_hathor_settings_fixture.yml @@ -2,7 +2,6 @@ P2PKH_VERSION_BYTE: x28 MULTISIG_VERSION_BYTE: '64' BOOTSTRAP_DNS: - mainnet.hathor.network -ENABLE_PEER_WHITELIST: true WHITELIST_URL: https://hathor-public-files.s3.amazonaws.com/whitelist_peer_ids GENESIS_OUTPUT_SCRIPT: 76a9147fd4ae0e4fb2d2854e76d359029d8078bb99649e88ac diff --git a/hathor_tests/others/fixtures/valid_hathor_settings_fixture.yml b/hathor_tests/others/fixtures/valid_hathor_settings_fixture.yml index b0198476f..617bf29d5 100644 --- a/hathor_tests/others/fixtures/valid_hathor_settings_fixture.yml +++ b/hathor_tests/others/fixtures/valid_hathor_settings_fixture.yml @@ -3,7 +3,6 @@ MULTISIG_VERSION_BYTE: '64' NETWORK_NAME: testing BOOTSTRAP_DNS: - mainnet.hathor.network -ENABLE_PEER_WHITELIST: true WHITELIST_URL: https://hathor-public-files.s3.amazonaws.com/whitelist_peer_ids GENESIS_OUTPUT_SCRIPT: 76a9147fd4ae0e4fb2d2854e76d359029d8078bb99649e88ac diff --git a/hathor_tests/others/test_cli_builder.py b/hathor_tests/others/test_cli_builder.py index 53925ed95..a2e4043e6 100644 --- a/hathor_tests/others/test_cli_builder.py +++ b/hathor_tests/others/test_cli_builder.py @@ -8,6 +8,7 @@ from hathor.indexes import RocksDBIndexesManager from hathor.manager import HathorManager from hathor.p2p.sync_version import SyncVersion +from hathor.p2p.whitelist import URLPeersWhitelist from hathor.transaction.storage import TransactionRocksDBStorage from hathor.wallet import HDWallet, Wallet from hathor_cli.builder import CliBuilder @@ -55,7 +56,6 @@ def test_all_default(self): self.assertIsInstance(manager.tx_storage.indexes, RocksDBIndexesManager) self.assertIsNone(manager.wallet) self.assertEqual('unittests', manager.network) - self.assertFalse(manager.connections.is_sync_version_enabled(SyncVersion.V1_1)) self.assertTrue(manager.connections.is_sync_version_enabled(SyncVersion.V2)) self.assertFalse(self.resources_builder._built_prometheus) self.assertFalse(self.resources_builder._built_status) @@ -76,7 +76,6 @@ def test_rocksdb_storage(self): def test_sync_default(self): manager = self._build(['--temp-data']) - self.assertFalse(manager.connections.is_sync_version_enabled(SyncVersion.V1_1)) self.assertTrue(manager.connections.is_sync_version_enabled(SyncVersion.V2)) def test_sync_bridge(self): @@ -87,12 +86,10 @@ def test_sync_bridge2(self): def test_sync_v2_only(self): manager = self._build(['--temp-data', '--x-sync-v2-only']) - self.assertFalse(manager.connections.is_sync_version_enabled(SyncVersion.V1_1)) self.assertTrue(manager.connections.is_sync_version_enabled(SyncVersion.V2)) def test_sync_v2_only2(self): manager = self._build(['--temp-data', '--sync-v2-only']) - self.assertFalse(manager.connections.is_sync_version_enabled(SyncVersion.V1_1)) self.assertTrue(manager.connections.is_sync_version_enabled(SyncVersion.V2)) def test_sync_v1_only(self): @@ -138,3 +135,21 @@ def test_event_queue_with_rocksdb_storage(self): self.assertIsInstance(manager._event_manager._event_storage, EventRocksDBStorage) self.assertIsInstance(manager._event_manager._event_ws_factory, EventWebsocketFactory) self.assertTrue(manager._enable_event_queue) + + def test_whitelist_cli_args(self): + """Test --x-p2p-whitelist and --x-p2p-whitelist-only CLI arguments.""" + # Test with whitelist URL + manager = self._build(['--temp-data', '--x-p2p-whitelist', 'https://example.com/whitelist']) + self.assertIsNotNone(manager.connections.peers_whitelist) + self.assertIsInstance(manager.connections.peers_whitelist, URLPeersWhitelist) + + # Test with whitelist-only flag (now a no-op, whitelist always enforces) + manager2 = self._build([ + '--temp-data', '--x-p2p-whitelist', 'https://example.com/whitelist', + '--x-p2p-whitelist-only', + ]) + self.assertIsNotNone(manager2.connections.peers_whitelist) + + # Test with disabled whitelist + manager3 = self._build(['--temp-data', '--x-p2p-whitelist', 'none']) + self.assertIsNone(manager3.connections.peers_whitelist) diff --git a/hathor_tests/others/test_hathor_settings.py b/hathor_tests/others/test_hathor_settings.py index 6fb718dc4..6d1be03a9 100644 --- a/hathor_tests/others/test_hathor_settings.py +++ b/hathor_tests/others/test_hathor_settings.py @@ -35,7 +35,6 @@ def test_valid_hathor_settings_from_yaml(filepath): MULTISIG_VERSION_BYTE=b'\x64', NETWORK_NAME='testing', BOOTSTRAP_DNS=['mainnet.hathor.network'], - ENABLE_PEER_WHITELIST=True, WHITELIST_URL='https://hathor-public-files.s3.amazonaws.com/whitelist_peer_ids', MIN_TX_WEIGHT_K=0, MIN_TX_WEIGHT_COEFFICIENT=0, diff --git a/hathor_tests/p2p/test_bootstrap.py b/hathor_tests/p2p/test_bootstrap.py index 55a9865bb..3bc206230 100644 --- a/hathor_tests/p2p/test_bootstrap.py +++ b/hathor_tests/p2p/test_bootstrap.py @@ -57,7 +57,7 @@ def test_mock_discovery(self) -> None: pubsub, True, self.rng, - True, + None, enable_ipv6=False, disable_ipv4=False ) @@ -92,7 +92,7 @@ def test_dns_discovery(self) -> None: pubsub, True, self.rng, - True, + None, enable_ipv6=False, disable_ipv4=False ) diff --git a/hathor_tests/p2p/test_capabilities.py b/hathor_tests/p2p/test_capabilities.py index e6957edcf..064a30e47 100644 --- a/hathor_tests/p2p/test_capabilities.py +++ b/hathor_tests/p2p/test_capabilities.py @@ -7,9 +7,24 @@ class CapabilitiesTestCase(unittest.TestCase): def test_capabilities(self) -> None: network = 'testnet' + url_1 = "https://whitelist1.com" + url_2 = "https://whitelist2.com" manager1 = self.create_peer(network, capabilities=[self._settings.CAPABILITY_WHITELIST, - self._settings.CAPABILITY_SYNC_VERSION]) - manager2 = self.create_peer(network, capabilities=[self._settings.CAPABILITY_SYNC_VERSION]) + self._settings.CAPABILITY_SYNC_VERSION], + url_whitelist=url_1) + manager2 = self.create_peer(network, capabilities=[self._settings.CAPABILITY_WHITELIST, + self._settings.CAPABILITY_SYNC_VERSION], + url_whitelist=url_2) + + assert manager1.connections.peers_whitelist is not None, 'Peers whitelist should not be None' + assert manager2.connections.peers_whitelist is not None, 'Peers whitelist should not be None' + assert len(manager1.connections.peers_whitelist._current) == 0, 'Should have no peers in the whitelist' + assert len(manager2.connections.peers_whitelist._current) == 0, 'Should have no peers in the whitelist' + + # Suspend whitelist to allow connections in test environment + # (empty whitelist + ONLY_WHITELISTED_PEERS policy would block all) + manager1.connections.set_peers_whitelist(None) + manager2.connections.set_peers_whitelist(None) conn = FakeConnection(manager1, manager2) @@ -18,29 +33,12 @@ def test_capabilities(self) -> None: conn.run_one_step(debug=True) self.clock.advance(0.1) - # Even if we don't have the capability we must connect because the whitelist url conf is None + # Update: Now, the URL has no effect to block the handle hello in HelloState - + # having capability or not is definitive to block the conn. + # Also, no more need to create two connections, one is enough to test the capabilities. assert isinstance(conn._proto1.state, ReadyState) assert isinstance(conn._proto2.state, ReadyState) self.assertEqual(conn._proto1.state.state_name, 'READY') self.assertEqual(conn._proto2.state.state_name, 'READY') self.assertIsInstance(conn._proto1.state.sync_agent, NodeBlockSync) self.assertIsInstance(conn._proto2.state.sync_agent, NodeBlockSync) - - manager3 = self.create_peer(network, capabilities=[self._settings.CAPABILITY_WHITELIST, - self._settings.CAPABILITY_SYNC_VERSION]) - manager4 = self.create_peer(network, capabilities=[self._settings.CAPABILITY_WHITELIST, - self._settings.CAPABILITY_SYNC_VERSION]) - - conn2 = FakeConnection(manager3, manager4) - - # Run the p2p protocol. - for _ in range(100): - conn2.run_one_step(debug=True) - self.clock.advance(0.1) - - assert isinstance(conn2._proto1.state, ReadyState) - assert isinstance(conn2._proto2.state, ReadyState) - self.assertEqual(conn2._proto1.state.state_name, 'READY') - self.assertEqual(conn2._proto2.state.state_name, 'READY') - self.assertIsInstance(conn2._proto1.state.sync_agent, NodeBlockSync) - self.assertIsInstance(conn2._proto2.state.sync_agent, NodeBlockSync) diff --git a/hathor_tests/p2p/test_protocol.py b/hathor_tests/p2p/test_protocol.py index 9c338af77..37b64715b 100644 --- a/hathor_tests/p2p/test_protocol.py +++ b/hathor_tests/p2p/test_protocol.py @@ -208,12 +208,14 @@ def test_hello_without_ipv6_capability(self) -> None: manager1 = self.create_peer( network, peer=self.peer1, - capabilities=[self._settings.CAPABILITY_IPV6, self._settings.CAPABILITY_SYNC_VERSION] + capabilities=[self._settings.CAPABILITY_WHITELIST, self._settings.CAPABILITY_IPV6, + self._settings.CAPABILITY_SYNC_VERSION] ) manager2 = self.create_peer( network, peer=self.peer2, - capabilities=[self._settings.CAPABILITY_SYNC_VERSION] + capabilities=[self._settings.CAPABILITY_WHITELIST, + self._settings.CAPABILITY_SYNC_VERSION] ) port1 = FakeConnection._get_port(manager1) @@ -248,12 +250,14 @@ def test_hello_with_ipv6_capability(self) -> None: manager1 = self.create_peer( network, peer=self.peer1, - capabilities=[self._settings.CAPABILITY_IPV6, self._settings.CAPABILITY_SYNC_VERSION] + capabilities=[self._settings.CAPABILITY_WHITELIST, self._settings.CAPABILITY_IPV6, + self._settings.CAPABILITY_SYNC_VERSION] ) manager2 = self.create_peer( network, peer=self.peer2, - capabilities=[self._settings.CAPABILITY_IPV6, self._settings.CAPABILITY_SYNC_VERSION] + capabilities=[self._settings.CAPABILITY_WHITELIST, + self._settings.CAPABILITY_IPV6, self._settings.CAPABILITY_SYNC_VERSION] ) port1 = FakeConnection._get_port(manager1) diff --git a/hathor_tests/p2p/test_whitelist.py b/hathor_tests/p2p/test_whitelist.py index 45e79639b..1c2fc68c7 100644 --- a/hathor_tests/p2p/test_whitelist.py +++ b/hathor_tests/p2p/test_whitelist.py @@ -1,3 +1,5 @@ +import tempfile +from typing import Any from unittest.mock import Mock, patch from twisted.internet.defer import Deferred, TimeoutError @@ -6,8 +8,21 @@ from hathor.conf.get_settings import get_global_settings from hathor.manager import HathorManager -from hathor.p2p.manager import WHITELIST_REQUEST_TIMEOUT +from hathor.p2p.peer_id import PeerId from hathor.p2p.sync_version import SyncVersion +from hathor.p2p.whitelist import ( + WHITELIST_REQUEST_TIMEOUT, + WHITELIST_RETRY_INTERVAL_MAX, + WHITELIST_RETRY_INTERVAL_MIN, + WHITELIST_SPEC_DEFAULT, + WHITELIST_SPEC_DISABLED, + WHITELIST_SPEC_HATHORLABS, + WHITELIST_SPEC_NONE, + FilePeersWhitelist, + URLPeersWhitelist, + WhitelistPolicy, + parse_whitelist_with_policy, +) from hathor.simulator import FakeConnection from hathor_tests import unittest @@ -15,14 +30,24 @@ class WhitelistTestCase(unittest.TestCase): def test_whitelist_no_no(self) -> None: network = 'testnet' - self._settings = get_global_settings().model_copy(update={'ENABLE_PEER_WHITELIST': True}) - - manager1 = self.create_peer(network) + self._settings = get_global_settings() + url_1 = 'https://whitelist1.com' + url_2 = 'https://whitelist2.com' + manager1 = self.create_peer(network, url_whitelist=url_1) self.assertEqual(manager1.connections.get_enabled_sync_versions(), {SyncVersion.V2}) - manager2 = self.create_peer(network) + manager2 = self.create_peer(network, url_whitelist=url_2) self.assertEqual(manager2.connections.get_enabled_sync_versions(), {SyncVersion.V2}) + # Create a dummy peer for both managers to populate their whitelists. + dummy_manager = self.create_peer(network) + manager1.connections.peers_whitelist.add_peer(dummy_manager.my_peer.id) + manager2.connections.peers_whitelist.add_peer(dummy_manager.my_peer.id) + + # Simulate successful fetches to end grace period + manager1.connections.peers_whitelist._has_successful_fetch = True + manager2.connections.peers_whitelist._has_successful_fetch = True + conn = FakeConnection(manager1, manager2) self.assertFalse(conn.tr1.disconnecting) self.assertFalse(conn.tr2.disconnecting) @@ -37,15 +62,28 @@ def test_whitelist_no_no(self) -> None: def test_whitelist_yes_no(self) -> None: network = 'testnet' - self._settings = get_global_settings().model_copy(update={'ENABLE_PEER_WHITELIST': True}) + url_1 = 'https://whitelist1.com' + url_2 = 'https://whitelist2.com' + self._settings = get_global_settings() + manager1 = self.create_peer(network, url_whitelist=url_1) - manager1 = self.create_peer(network) self.assertEqual(manager1.connections.get_enabled_sync_versions(), {SyncVersion.V2}) - manager2 = self.create_peer(network) + manager2 = self.create_peer(network, url_whitelist=url_2) + # Both follow their respective whitelist, although manager1 is not in manager2's whitelist. self.assertEqual(manager2.connections.get_enabled_sync_versions(), {SyncVersion.V2}) - manager1.peers_whitelist.append(manager2.my_peer.id) + # Whitelist of Manager 2 is empty, which still lets connections happen. + # We'll create a dummy peer id for manager2 to simulate a whitelist entry. + dummy_manager = self.create_peer(network) + manager2.connections.peers_whitelist.add_peer(dummy_manager.my_peer.id) + + # Now, manager2 has a non-empty whitelist, so not having manager1 in it will cause a disconnect. + manager1.connections.peers_whitelist.add_peer(manager2.my_peer.id) + + # Simulate successful fetches to end grace period + manager1.connections.peers_whitelist._has_successful_fetch = True + manager2.connections.peers_whitelist._has_successful_fetch = True conn = FakeConnection(manager1, manager2) self.assertFalse(conn.tr1.disconnecting) @@ -61,16 +99,28 @@ def test_whitelist_yes_no(self) -> None: def test_whitelist_yes_yes(self) -> None: network = 'testnet' - self._settings = get_global_settings().model_copy(update={'ENABLE_PEER_WHITELIST': True}) - - manager1 = self.create_peer(network) + self._settings = get_global_settings() + url_1 = 'https://whitelist1.com' + url_2 = 'https://whitelist2.com' + manager1 = self.create_peer(network, url_whitelist=url_1) self.assertEqual(manager1.connections.get_enabled_sync_versions(), {SyncVersion.V2}) - manager2 = self.create_peer(network) + manager2 = self.create_peer(network, url_whitelist=url_2) self.assertEqual(manager2.connections.get_enabled_sync_versions(), {SyncVersion.V2}) - manager1.peers_whitelist.append(manager2.my_peer.id) - manager2.peers_whitelist.append(manager1.my_peer.id) + # Mock Peers Whitelist does not fetch peer Ids from blank url + self.assertTrue(manager1.connections.peers_whitelist.current_whitelist() == set()) + self.assertTrue(manager2.connections.peers_whitelist.current_whitelist() == set()) + + manager1.connections.peers_whitelist.add_peer(manager2.my_peer.id) + manager2.connections.peers_whitelist.add_peer(manager1.my_peer.id) + + self.assertTrue(len(manager1.connections.peers_whitelist.current_whitelist()) == 1) + self.assertTrue(len(manager2.connections.peers_whitelist.current_whitelist()) == 1) + + # Simulate successful fetches to end grace period + manager1.connections.peers_whitelist._has_successful_fetch = True + manager2.connections.peers_whitelist._has_successful_fetch = True conn = FakeConnection(manager1, manager2) self.assertFalse(conn.tr1.disconnecting) @@ -90,22 +140,24 @@ def test_update_whitelist(self) -> None: connections_manager = manager.connections settings_mock = Mock() - settings_mock.WHITELIST_URL = 'some_url' + settings_mock.WHITELIST_URL = 'https://something.com' connections_manager._settings = settings_mock agent_mock = Mock(spec_set=Agent) agent_mock.request = Mock() - connections_manager._http_agent = agent_mock + if type(connections_manager.peers_whitelist) is not URLPeersWhitelist: + return + connections_manager.peers_whitelist._http_agent = agent_mock with ( - patch.object(connections_manager, '_update_whitelist_cb') as _update_whitelist_cb_mock, - patch.object(connections_manager, '_update_whitelist_err') as _update_whitelist_err_mock, + patch.object(connections_manager.peers_whitelist, '_update_whitelist_cb') as _update_whitelist_cb_mock, + patch.object(connections_manager.peers_whitelist, '_update_whitelist_err') as _update_whitelist_err_mock, patch('twisted.web.client.readBody') as read_body_mock ): # Test success agent_mock.request.return_value = Deferred() read_body_mock.return_value = b'body' - d = connections_manager.update_whitelist() + d = connections_manager.peers_whitelist.update() d.callback(None) read_body_mock.assert_called_once_with(None) @@ -118,7 +170,7 @@ def test_update_whitelist(self) -> None: # Test request error agent_mock.request.return_value = Deferred() - d = connections_manager.update_whitelist() + d = connections_manager.peers_whitelist.update() error = Failure('some_error') d.errback(error) @@ -133,11 +185,1000 @@ def test_update_whitelist(self) -> None: # Test timeout agent_mock.request.return_value = Deferred() read_body_mock.return_value = b'body' - connections_manager.update_whitelist() + connections_manager.peers_whitelist.update() self.clock.advance(WHITELIST_REQUEST_TIMEOUT + 1) read_body_mock.assert_not_called() _update_whitelist_cb_mock.assert_not_called() _update_whitelist_err_mock.assert_called_once() + # Check final instance assert isinstance(_update_whitelist_err_mock.call_args.args[0].value, TimeoutError) + + def test_empty_whitelist_blocks_peers(self) -> None: + """Test that empty whitelist with ONLY_WHITELISTED_PEERS policy blocks peers after grace period. + + With the fix for the empty whitelist policy bug, an empty whitelist with + restrictive policy should now block all peers (not allow all as before), + but only after the grace period ends (first successful fetch). + """ + network = 'testnet' + self._settings = get_global_settings() + url_1 = 'https://whitelist1.com' + url_2 = 'https://whitelist2.com' + manager1 = self.create_peer(network, url_whitelist=url_1) + self.assertEqual(manager1.connections.get_enabled_sync_versions(), {SyncVersion.V2}) + + manager2 = self.create_peer(network, url_whitelist=url_2) + self.assertEqual(manager2.connections.get_enabled_sync_versions(), {SyncVersion.V2}) + + # No peers will be added to the whitelist, so _current is empty. + # With ONLY_WHITELISTED_PEERS policy, empty whitelist should block all peers after grace period. + + self.assertTrue(len(manager1.connections.peers_whitelist.current_whitelist()) == 0) + self.assertTrue(len(manager2.connections.peers_whitelist.current_whitelist()) == 0) + + # Simulate successful fetches to end grace period + manager1.connections.peers_whitelist._has_successful_fetch = True + manager2.connections.peers_whitelist._has_successful_fetch = True + + conn = FakeConnection(manager1, manager2) + self.assertFalse(conn.tr1.disconnecting) + self.assertFalse(conn.tr2.disconnecting) + + # Run the p2p protocol. + for _ in range(100): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + # With empty whitelist and restrictive policy, peers should be blocked + self.assertTrue(conn.tr1.disconnecting or conn.tr2.disconnecting) + + +class ParseWhitelistWithPolicyTestCase(unittest.TestCase): + """Tests for parse_whitelist_with_policy function.""" + + def test_parse_allow_all_policy(self) -> None: + """Test parsing whitelist with allow-all policy.""" + content = """hathor-whitelist +policy: allow-all +""" + peers, policy = parse_whitelist_with_policy(content) + self.assertEqual(policy, WhitelistPolicy.ALLOW_ALL) + self.assertEqual(peers, set()) + + def test_parse_only_whitelisted_peers_policy(self) -> None: + """Test parsing whitelist with only-whitelisted-peers policy.""" + content = """hathor-whitelist +policy: only-whitelisted-peers +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +""" + peers, policy = parse_whitelist_with_policy(content) + self.assertEqual(policy, WhitelistPolicy.ONLY_WHITELISTED_PEERS) + self.assertEqual(len(peers), 1) + self.assertIn(PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'), peers) + + def test_parse_default_policy(self) -> None: + """Test that default policy is ONLY_WHITELISTED_PEERS when no policy line.""" + content = """hathor-whitelist +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +""" + peers, policy = parse_whitelist_with_policy(content) + self.assertEqual(policy, WhitelistPolicy.ONLY_WHITELISTED_PEERS) + self.assertEqual(len(peers), 1) + + def test_parse_policy_with_comments(self) -> None: + """Test parsing whitelist with policy and comments.""" + content = """hathor-whitelist +# This whitelist allows all peers +policy: allow-all +# More comments here +""" + peers, policy = parse_whitelist_with_policy(content) + self.assertEqual(policy, WhitelistPolicy.ALLOW_ALL) + self.assertEqual(peers, set()) + + def test_parse_policy_after_peer_raises_error(self) -> None: + """Test that policy line after peer IDs raises ValueError.""" + content = """hathor-whitelist +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +policy: allow-all +""" + with self.assertRaises(ValueError) as context: + parse_whitelist_with_policy(content) + self.assertIn('policy must be defined in the header', str(context.exception)) + + def test_parse_invalid_policy(self) -> None: + """Test that invalid policy raises ValueError.""" + content = """hathor-whitelist +policy: invalid-policy +""" + with self.assertRaises(ValueError) as context: + parse_whitelist_with_policy(content) + self.assertIn('invalid whitelist policy', str(context.exception)) + + def test_parse_policy_case_insensitive(self) -> None: + """Test that policy values are case-insensitive.""" + content = """hathor-whitelist +policy: ALLOW-ALL +""" + peers, policy = parse_whitelist_with_policy(content) + self.assertEqual(policy, WhitelistPolicy.ALLOW_ALL) + + +class WhitelistPolicyBehaviorTestCase(unittest.TestCase): + """Tests for WhitelistPolicy behavior in PeersWhitelist.""" + + def test_is_peer_whitelisted_with_allow_all(self) -> None: + """Test that is_peer_whitelisted returns True with ALLOW_ALL policy after grace period.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + # Set policy to ALLOW_ALL + whitelist._policy = WhitelistPolicy.ALLOW_ALL + + # Simulate successful fetch to end grace period + whitelist._has_successful_fetch = True + + # Even with an empty whitelist, any peer should be allowed with ALLOW_ALL policy + random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + self.assertTrue(whitelist.is_peer_whitelisted(random_peer_id)) + + def test_is_peer_whitelisted_with_only_whitelisted_peers(self) -> None: + """Test that is_peer_whitelisted checks whitelist with ONLY_WHITELISTED_PEERS policy after grace period.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + # Ensure policy is ONLY_WHITELISTED_PEERS (default) + self.assertEqual(whitelist.policy(), WhitelistPolicy.ONLY_WHITELISTED_PEERS) + + # Simulate a successful fetch to end grace period + whitelist._has_successful_fetch = True + + # Add a peer to whitelist + whitelisted_peer_id = PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367') + whitelist.add_peer(whitelisted_peer_id) + + # Whitelisted peer should be allowed + self.assertTrue(whitelist.is_peer_whitelisted(whitelisted_peer_id)) + + # Non-whitelisted peer should not be allowed after grace period + random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + + def test_policy_method(self) -> None: + """Test the policy() method returns correct policy.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + # Default policy + self.assertEqual(whitelist.policy(), WhitelistPolicy.ONLY_WHITELISTED_PEERS) + + # Change policy + whitelist._policy = WhitelistPolicy.ALLOW_ALL + self.assertEqual(whitelist.policy(), WhitelistPolicy.ALLOW_ALL) + + +class FilePeersWhitelistTestCase(unittest.TestCase): + """Tests for FilePeersWhitelist class.""" + + def test_file_whitelist_reads_valid_file(self) -> None: + """Test that FilePeersWhitelist correctly reads a valid whitelist file.""" + content = """hathor-whitelist +policy: only-whitelisted-peers +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(content) + f.flush() + path = f.name + + whitelist = FilePeersWhitelist(self.clock, path) + + # Mock deferToThread to call the function directly (synchronously) + with patch('hathor.p2p.whitelist.file_whitelist.threads.deferToThread') as mock_defer: + def call_directly(func: Any, *args: Any, **kwargs: Any) -> Deferred[None]: + d: Deferred[None] = Deferred() + try: + result = func(*args, **kwargs) + d.callback(result) + except Exception as e: + d.errback(e) + return d + mock_defer.side_effect = call_directly + + whitelist.update() + + # Verify the whitelist was parsed correctly + self.assertEqual(len(whitelist.current_whitelist()), 2) + self.assertEqual(whitelist.policy(), WhitelistPolicy.ONLY_WHITELISTED_PEERS) + peer_id = PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367') + self.assertIn(peer_id, whitelist.current_whitelist()) + + def test_file_whitelist_handles_missing_file(self) -> None: + """Test that FilePeersWhitelist handles missing files gracefully.""" + whitelist = FilePeersWhitelist(self.clock, '/nonexistent/path/whitelist.txt') + + # Mock deferToThread to call the function directly (synchronously) + with patch('hathor.p2p.whitelist.file_whitelist.threads.deferToThread') as mock_defer: + def call_directly(func: Any, *args: Any, **kwargs: Any) -> Deferred[None]: + d: Deferred[None] = Deferred() + try: + result = func(*args, **kwargs) + d.callback(result) + except Exception as e: + d.errback(Failure(e)) + return d + mock_defer.side_effect = call_directly + + whitelist.update() + + # Whitelist should remain empty + self.assertEqual(len(whitelist.current_whitelist()), 0) + # Failure counter should be incremented + self.assertEqual(whitelist._consecutive_failures, 1) + + def test_file_whitelist_handles_permission_error(self) -> None: + """Test that FilePeersWhitelist handles permission errors gracefully.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write("hathor-whitelist\n") + path = f.name + + whitelist = FilePeersWhitelist(self.clock, path) + + # Mock deferToThread to raise PermissionError + with patch('hathor.p2p.whitelist.file_whitelist.threads.deferToThread') as mock_defer: + def raise_permission_error(func: Any, *args: Any, **kwargs: Any) -> Deferred[None]: + d: Deferred[None] = Deferred() + d.errback(Failure(PermissionError("Permission denied"))) + return d + mock_defer.side_effect = raise_permission_error + + whitelist.update() + + # Whitelist should remain empty + self.assertEqual(len(whitelist.current_whitelist()), 0) + # Failure counter should be incremented + self.assertEqual(whitelist._consecutive_failures, 1) + + def test_file_whitelist_refresh_returns_deferred(self) -> None: + """Test that FilePeersWhitelist.refresh() returns a Deferred.""" + content = """hathor-whitelist +policy: allow-all +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(content) + f.flush() + path = f.name + + whitelist = FilePeersWhitelist(self.clock, path) + result = whitelist.refresh() + + self.assertIsInstance(result, Deferred) + + +class InvalidPeerIdParsingTestCase(unittest.TestCase): + """Tests for invalid peer ID handling in whitelist parsing.""" + + def test_parse_whitelist_with_invalid_hex(self) -> None: + """Test that invalid hex characters in peer ID are skipped.""" + content = """hathor-whitelist +# Valid peer +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +# Invalid peer (contains 'G') +Gffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +""" + peers, policy = parse_whitelist_with_policy(content) + # Only the valid peer should be parsed + self.assertEqual(len(peers), 1) + self.assertIn(PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'), peers) + + def test_parse_whitelist_with_short_id(self) -> None: + """Test that short peer IDs are skipped.""" + content = """hathor-whitelist +# Valid peer +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +# Too short +2ffdfbbfd6d869a0 +""" + peers, policy = parse_whitelist_with_policy(content) + # Only the valid peer should be parsed + self.assertEqual(len(peers), 1) + + def test_parse_whitelist_skips_invalid_keeps_valid(self) -> None: + """Test that invalid peer IDs are skipped while valid ones are kept.""" + content = """hathor-whitelist +# Valid peer 1 +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +# Invalid peer +not-a-valid-peer-id +# Valid peer 2 +1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef +# Another invalid one +xyz +""" + peers, policy = parse_whitelist_with_policy(content) + # Only the valid peers should be parsed + self.assertEqual(len(peers), 2) + self.assertIn(PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'), peers) + self.assertIn(PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'), peers) + + +class ExponentialBackoffTestCase(unittest.TestCase): + """Tests for exponential backoff retry mechanism.""" + + def test_url_whitelist_retry_backoff(self) -> None: + """Test that URL whitelist uses exponential backoff on failures.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + if not isinstance(whitelist, URLPeersWhitelist): + self.skipTest("Test requires URLPeersWhitelist") + + # Initial state + self.assertEqual(whitelist._consecutive_failures, 0) + + # Simulate failures + whitelist._on_update_failure() + self.assertEqual(whitelist._consecutive_failures, 1) + self.assertEqual(whitelist._get_retry_interval(), WHITELIST_RETRY_INTERVAL_MIN * 2) + + whitelist._on_update_failure() + self.assertEqual(whitelist._consecutive_failures, 2) + self.assertEqual(whitelist._get_retry_interval(), WHITELIST_RETRY_INTERVAL_MIN * 4) + + whitelist._on_update_failure() + self.assertEqual(whitelist._consecutive_failures, 3) + self.assertEqual(whitelist._get_retry_interval(), WHITELIST_RETRY_INTERVAL_MIN * 8) + + # Test max cap + for _ in range(10): + whitelist._on_update_failure() + self.assertEqual(whitelist._get_retry_interval(), WHITELIST_RETRY_INTERVAL_MAX) + + # Test reset on success + whitelist._on_update_success() + self.assertEqual(whitelist._consecutive_failures, 0) + self.assertEqual(whitelist._get_retry_interval(), WHITELIST_RETRY_INTERVAL_MIN) + + def test_file_whitelist_retry_backoff(self) -> None: + """Test that file whitelist uses exponential backoff on failures.""" + whitelist = FilePeersWhitelist(self.clock, '/nonexistent/path.txt') + + # Initial state + self.assertEqual(whitelist._consecutive_failures, 0) + self.assertEqual(whitelist._get_retry_interval(), WHITELIST_RETRY_INTERVAL_MIN) + + # Simulate failures + whitelist._on_update_failure() + self.assertEqual(whitelist._consecutive_failures, 1) + # After first failure, interval doubles + self.assertEqual(whitelist._get_retry_interval(), WHITELIST_RETRY_INTERVAL_MIN * 2) + + +class SysctlWhitelistTestCase(unittest.TestCase): + """Tests for sysctl whitelist toggle operations.""" + + def test_sysctl_toggle_on_without_whitelist_error(self) -> None: + """Test that set_peers_whitelist handles case when no whitelist is set.""" + network = 'testnet' + manager = self.create_peer(network) # no whitelist + + # This should not raise an error + manager.connections.set_peers_whitelist(None) + manager.connections.set_peers_whitelist(None) + + def test_sysctl_toggle_disconnects_non_whitelisted(self) -> None: + """Test that toggling whitelist ON disconnects non-whitelisted peers.""" + network = 'testnet' + manager1 = self.create_peer(network, url_whitelist='https://whitelist.com') + manager2 = self.create_peer(network, url_whitelist='https://whitelist.com') + + # Save whitelist references before suspending + saved_whitelist1 = manager1.connections.peers_whitelist + saved_whitelist2 = manager2.connections.peers_whitelist + + # Suspend whitelist during connection setup to allow peers to connect + manager1.connections.set_peers_whitelist(None) + manager2.connections.set_peers_whitelist(None) + + # Simulate successful fetches to end grace period (so whitelist rules apply when enabled) + saved_whitelist1._has_successful_fetch = True + saved_whitelist2._has_successful_fetch = True + + # Connect the peers + conn = FakeConnection(manager1, manager2) + for _ in range(100): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + # Both should be connected + self.assertFalse(conn.tr1.disconnecting) + self.assertFalse(conn.tr2.disconnecting) + + # Add manager2 to manager1's saved whitelist then re-enable + saved_whitelist1.add_peer(manager2.my_peer.id) + + # Turn on whitelist on manager1 (should do nothing since manager2 is whitelisted) + manager1.connections.set_peers_whitelist(saved_whitelist1) + + # Run some steps + for _ in range(10): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + # Connections should still be up (manager2 is in manager1's whitelist) + self.assertFalse(conn.tr1.disconnecting) + + def test_sysctl_swap_to_file(self) -> None: + """Test swapping from URL whitelist to file whitelist.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + + # Create a temporary file whitelist + content = """hathor-whitelist +policy: allow-all +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(content) + f.flush() + path = f.name + + # Create a file whitelist + file_whitelist = FilePeersWhitelist(self.clock, path) + + # Swap whitelists + manager.connections.set_peers_whitelist(file_whitelist) + + # Verify the swap + self.assertIsInstance(manager.connections.peers_whitelist, FilePeersWhitelist) + + def test_sysctl_swap_to_url(self) -> None: + """Test swapping from file whitelist to URL whitelist.""" + # Create a manager with a URL whitelist first + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://oldwhitelist.com') + + # Create a new URL whitelist + new_url_whitelist = URLPeersWhitelist(self.clock, 'https://newwhitelist.com') + + # Swap whitelists + manager.connections.set_peers_whitelist(new_url_whitelist) + + # Verify the swap + self.assertIsInstance(manager.connections.peers_whitelist, URLPeersWhitelist) + self.assertEqual(manager.connections.peers_whitelist.url(), 'https://newwhitelist.com') + + +class RaceConditionTestCase(unittest.TestCase): + """Tests for race condition fixes.""" + + def test_race_condition_whitelist_toggle(self) -> None: + """Test that whitelist_update uses a snapshot to avoid race conditions.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + + # Create mock connections + mock_conns = [] + for i in range(5): + mock_conn = Mock() + mock_conn.get_peer_id.return_value = PeerId(f'{i:064x}') + mock_conns.append(mock_conn) + + # Set up the connections set + manager.connections.connections = set(mock_conns) + + # Add only the first peer to whitelist + manager.connections.peers_whitelist.add_peer(PeerId(f'{0:064x}')) + + # Simulate a successful whitelist fetch to end grace period + manager.connections.peers_whitelist._has_successful_fetch = True + + # Trigger disconnect of non-whitelisted peers + manager.connections._disconnect_non_whitelisted_peers() + + # The first connection should NOT be disconnected (it's whitelisted) + mock_conns[0].disconnect.assert_not_called() + + # The other connections should be disconnected + for i in range(1, 5): + mock_conns[i].disconnect.assert_called_once() + + +class URLValidationTestCase(unittest.TestCase): + """Tests for URL validation in URLPeersWhitelist.""" + + def test_url_whitelist_none_string(self) -> None: + """Test that 'none' string URL is converted to None.""" + whitelist = URLPeersWhitelist(self.clock, 'none', mainnet=False) + self.assertIsNone(whitelist.url()) + + def test_url_whitelist_none_string_case_insensitive(self) -> None: + """Test that 'NONE' string URL is converted to None (case insensitive).""" + whitelist = URLPeersWhitelist(self.clock, 'NONE', mainnet=False) + self.assertIsNone(whitelist.url()) + + def test_url_whitelist_actual_none(self) -> None: + """Test that actual None URL works.""" + whitelist = URLPeersWhitelist(self.clock, None, mainnet=False) + self.assertIsNone(whitelist.url()) + + def test_url_whitelist_mainnet_requires_https(self) -> None: + """Test that mainnet requires HTTPS URLs.""" + with self.assertRaises(ValueError) as context: + URLPeersWhitelist(self.clock, 'http://whitelist.com', mainnet=True) + self.assertIn('invalid scheme', str(context.exception)) + + def test_url_whitelist_mainnet_accepts_https(self) -> None: + """Test that mainnet accepts HTTPS URLs.""" + whitelist = URLPeersWhitelist(self.clock, 'https://whitelist.com', mainnet=True) + self.assertEqual(whitelist.url(), 'https://whitelist.com') + + def test_url_whitelist_none_url_does_not_crash_on_update(self) -> None: + """Test that URLPeersWhitelist with url=None does not crash on update.""" + whitelist = URLPeersWhitelist(self.clock, 'none', mainnet=False) + self.assertIsNone(whitelist.url()) + + # This should not crash - it should return early + d = whitelist.update() + + # Verify that no crash occurred and the deferred completed + self.assertIsNotNone(d) + # The deferred should complete immediately since URL is None + # Verify failure count wasn't incremented (no actual update attempted) + self.assertEqual(whitelist._consecutive_failures, 0) + + +class EmptyWhitelistPolicyTestCase(unittest.TestCase): + """Tests for empty whitelist behavior with different policies.""" + + def test_empty_whitelist_with_only_whitelisted_peers_blocks_all(self) -> None: + """Test that empty whitelist with ONLY_WHITELISTED_PEERS policy blocks all peers after grace period.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + # Ensure whitelist has ONLY_WHITELISTED_PEERS policy (default) + self.assertEqual(whitelist.policy(), WhitelistPolicy.ONLY_WHITELISTED_PEERS) + + # Ensure whitelist is empty + self.assertEqual(len(whitelist.current_whitelist()), 0) + + # Simulate a successful fetch to end grace period + whitelist._has_successful_fetch = True + + # Any peer should NOT be whitelisted with empty list + restrictive policy after grace period + random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + + def test_empty_whitelist_with_allow_all_allows_all(self) -> None: + """Test that empty whitelist with ALLOW_ALL policy allows all peers after grace period.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + # Set policy to ALLOW_ALL + whitelist._policy = WhitelistPolicy.ALLOW_ALL + + # Simulate successful fetch to end grace period + whitelist._has_successful_fetch = True + + # Ensure whitelist is empty + self.assertEqual(len(whitelist.current_whitelist()), 0) + + # Any peer should be whitelisted with ALLOW_ALL policy + random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + self.assertTrue(whitelist.is_peer_whitelisted(random_peer_id)) + + def test_empty_whitelist_blocks_connection_with_restrictive_policy(self) -> None: + """Test that empty whitelist blocks connections when policy is ONLY_WHITELISTED_PEERS after grace period.""" + network = 'testnet' + manager1 = self.create_peer(network, url_whitelist='https://whitelist1.com') + manager2 = self.create_peer(network, url_whitelist='https://whitelist2.com') + + # Simulate successful fetches to end grace period + manager1.connections.peers_whitelist._has_successful_fetch = True + manager2.connections.peers_whitelist._has_successful_fetch = True + + # Verify whitelists are empty + self.assertEqual(len(manager1.connections.peers_whitelist.current_whitelist()), 0) + self.assertEqual(len(manager2.connections.peers_whitelist.current_whitelist()), 0) + + conn = FakeConnection(manager1, manager2) + + # Run the p2p protocol + for _ in range(100): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + # Both connections should be disconnected because neither peer is in the other's whitelist + self.assertTrue(conn.tr1.disconnecting or conn.tr2.disconnecting) + + +class WhitelistToggleNullPeerIdTestCase(unittest.TestCase): + """Tests for whitelist_update handling of None peer_id.""" + + def test_whitelist_toggle_handles_none_peer_id(self) -> None: + """Test that whitelist_update handles connections with get_peer_id()=None.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + + # Create mock connections - some with peer_id, some without + mock_conns = [] + + # Connection with peer_id + mock_conn_with_id = Mock() + mock_conn_with_id.get_peer_id.return_value = PeerId(f'{1:064x}') + mock_conns.append(mock_conn_with_id) + + # Connection without peer_id (still handshaking) + mock_conn_without_id = Mock() + mock_conn_without_id.get_peer_id.return_value = None + mock_conns.append(mock_conn_without_id) + + # Another connection with peer_id + mock_conn_with_id2 = Mock() + mock_conn_with_id2.get_peer_id.return_value = PeerId(f'{2:064x}') + mock_conns.append(mock_conn_with_id2) + + # Set up the connections set + manager.connections.connections = set(mock_conns) + + # Simulate successful fetch to end grace period + manager.connections.peers_whitelist._has_successful_fetch = True + + # Don't add any peer to whitelist (empty whitelist with restrictive policy) + # This should disconnect peers with peer_id but skip those without + + # Trigger disconnect of non-whitelisted peers - this should NOT crash even with None peer_id + manager.connections._disconnect_non_whitelisted_peers() + + # Connection without peer_id should NOT have disconnect called + mock_conn_without_id.disconnect.assert_not_called() + + # Connections with peer_id should have disconnect called (they're not in whitelist) + mock_conn_with_id.disconnect.assert_called_once() + mock_conn_with_id2.disconnect.assert_called_once() + + +class WhitelistLifecycleTestCase(unittest.TestCase): + """Integration tests for full whitelist lifecycle.""" + + def test_whitelist_lifecycle_add_peer_then_remove(self) -> None: + """Test full lifecycle: connect → add to whitelist → enable → remove → disconnect.""" + network = 'testnet' + + # Create two peers with whitelists but suspended initially + manager1 = self.create_peer(network, url_whitelist='https://whitelist1.com') + manager2 = self.create_peer(network, url_whitelist='https://whitelist2.com') + + # Save whitelist references before clearing + saved_whitelist1 = manager1.connections.peers_whitelist + saved_whitelist2 = manager2.connections.peers_whitelist + + # Suspend whitelists to allow initial connection + manager1.connections.set_peers_whitelist(None) + manager2.connections.set_peers_whitelist(None) + + # Connect the peers + conn = FakeConnection(manager1, manager2) + for _ in range(100): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + # Both should be connected + self.assertFalse(conn.tr1.disconnecting) + self.assertFalse(conn.tr2.disconnecting) + self.assertTrue(manager1.connections.is_peer_connected(manager2.my_peer.id)) + self.assertTrue(manager2.connections.is_peer_connected(manager1.my_peer.id)) + + # Add each peer to the saved whitelist references + saved_whitelist1.add_peer(manager2.my_peer.id) + saved_whitelist2.add_peer(manager1.my_peer.id) + + # Simulate successful fetches to end grace period + saved_whitelist1._has_successful_fetch = True + saved_whitelist2._has_successful_fetch = True + + # Re-enable whitelists + manager1.connections.set_peers_whitelist(saved_whitelist1) + manager2.connections.set_peers_whitelist(saved_whitelist2) + + # Run some steps - connections should remain up (peers are whitelisted) + for _ in range(10): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + self.assertFalse(conn.tr1.disconnecting) + self.assertFalse(conn.tr2.disconnecting) + + # Now simulate removing manager2 from manager1's whitelist + # Create a new whitelist without manager2 + manager1.connections.peers_whitelist._current = set() + + # Trigger disconnection of non-whitelisted peers + manager1.connections._disconnect_non_whitelisted_peers() + + # manager2 should be disconnected from manager1's perspective + # (manager1 initiated the disconnect) + self.assertTrue(conn.tr1.disconnecting or conn.tr2.disconnecting) + + def test_whitelist_policy_change_affects_connections(self) -> None: + """Test that changing whitelist policy affects existing connections.""" + network = 'testnet' + + manager1 = self.create_peer(network, url_whitelist='https://whitelist1.com') + manager2 = self.create_peer(network, url_whitelist='https://whitelist2.com') + + # Set policy to ALLOW_ALL so connections work even with empty whitelist + manager1.connections.peers_whitelist._policy = WhitelistPolicy.ALLOW_ALL + manager2.connections.peers_whitelist._policy = WhitelistPolicy.ALLOW_ALL + + # Simulate successful fetches to end grace period + manager1.connections.peers_whitelist._has_successful_fetch = True + manager2.connections.peers_whitelist._has_successful_fetch = True + + # Connect the peers - should work with ALLOW_ALL policy + conn = FakeConnection(manager1, manager2) + for _ in range(100): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + # Both should be connected + self.assertFalse(conn.tr1.disconnecting) + self.assertFalse(conn.tr2.disconnecting) + + # Change policy to ONLY_WHITELISTED_PEERS (whitelist is still empty) + manager1.connections.peers_whitelist._policy = WhitelistPolicy.ONLY_WHITELISTED_PEERS + + # Trigger disconnect of non-whitelisted peers + manager1.connections._disconnect_non_whitelisted_peers() + + # manager2 should be disconnected (not in whitelist with restrictive policy) + self.assertTrue(conn.tr1.disconnecting or conn.tr2.disconnecting) + + def test_whitelist_source_method(self) -> None: + """Test that source() method returns correct values for different whitelist types.""" + # Test URLPeersWhitelist + url_whitelist = URLPeersWhitelist(self.clock, 'https://example.com/whitelist', mainnet=False) + self.assertEqual(url_whitelist.source(), 'https://example.com/whitelist') + + # Test URLPeersWhitelist with None URL + none_whitelist = URLPeersWhitelist(self.clock, 'none', mainnet=False) + self.assertIsNone(none_whitelist.source()) + + # Test FilePeersWhitelist + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write("hathor-whitelist\npolicy: allow-all\n") + f.flush() + path = f.name + + file_whitelist = FilePeersWhitelist(self.clock, path) + self.assertEqual(file_whitelist.source(), path) + + +class GracePeriodTestCase(unittest.TestCase): + """Tests for grace period behavior before first successful whitelist fetch.""" + + def test_grace_period_rejects_non_bootstrap_peers_before_first_fetch(self) -> None: + """Test that non-bootstrap peers are rejected before first successful whitelist fetch.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + # Verify initial state: no successful fetch yet + self.assertFalse(whitelist._has_successful_fetch) + + self.assertEqual(whitelist.policy(), WhitelistPolicy.ONLY_WHITELISTED_PEERS) + + # Random peers should NOT be allowed during grace period (only bootstrap peers allowed) + random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + + def test_grace_period_allows_bootstrap_peers_before_first_fetch(self) -> None: + """Test that bootstrap peers are allowed before first successful whitelist fetch.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + # Verify initial state: no successful fetch yet + self.assertFalse(whitelist._has_successful_fetch) + + self.assertEqual(whitelist.policy(), WhitelistPolicy.ONLY_WHITELISTED_PEERS) + + # Register a bootstrap peer + bootstrap_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + whitelist.add_bootstrap_peer(bootstrap_peer_id) + + # Bootstrap peer should be allowed during grace period + self.assertTrue(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + + # Non-bootstrap peer should still be rejected + random_peer_id = PeerId('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890') + self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + + def test_grace_period_ends_after_successful_fetch(self) -> None: + """Test that grace period ends after first successful whitelist fetch.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + # Verify initial state: no successful fetch yet + self.assertFalse(whitelist._has_successful_fetch) + + # Simulate a successful whitelist update + whitelisted_peer_id = PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367') + whitelist._apply_whitelist_update({whitelisted_peer_id}, WhitelistPolicy.ONLY_WHITELISTED_PEERS) + + # Verify grace period has ended + self.assertTrue(whitelist._has_successful_fetch) + + # Now the whitelist should enforce - whitelisted peer is allowed + self.assertTrue(whitelist.is_peer_whitelisted(whitelisted_peer_id)) + + # Non-whitelisted peer should be blocked after grace period + random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + + def test_bootstrap_peer_allowed_during_grace_period_but_normal_rules_after(self) -> None: + """Test that bootstrap peers follow normal whitelist rules after successful fetch.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist.com') + whitelist = manager.connections.peers_whitelist + + # Verify initial state: no successful fetch yet + self.assertFalse(whitelist._has_successful_fetch) + + # Register a bootstrap peer + bootstrap_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + whitelist.add_bootstrap_peer(bootstrap_peer_id) + + # Bootstrap peer should be allowed during grace period + self.assertTrue(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + + # Simulate a successful whitelist update that does NOT include the bootstrap peer + other_peer_id = PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367') + whitelist._apply_whitelist_update({other_peer_id}, WhitelistPolicy.ONLY_WHITELISTED_PEERS) + + # Verify grace period has ended + self.assertTrue(whitelist._has_successful_fetch) + + # Bootstrap peer should NOT be allowed anymore (not in whitelist) + self.assertFalse(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + + # Peer in whitelist should be allowed + self.assertTrue(whitelist.is_peer_whitelisted(other_peer_id)) + + def test_grace_period_connections_rejected_without_bootstrap(self) -> None: + """Test that connections are rejected during grace period without bootstrap registration.""" + network = 'testnet' + manager1 = self.create_peer(network, url_whitelist='https://whitelist1.com') + manager2 = self.create_peer(network, url_whitelist='https://whitelist2.com') + + # Verify both whitelists have not had successful fetches + self.assertFalse(manager1.connections.peers_whitelist._has_successful_fetch) + self.assertFalse(manager2.connections.peers_whitelist._has_successful_fetch) + + # Whitelists are empty and no bootstrap peers registered + self.assertEqual(len(manager1.connections.peers_whitelist.current_whitelist()), 0) + self.assertEqual(len(manager2.connections.peers_whitelist.current_whitelist()), 0) + + conn = FakeConnection(manager1, manager2) + + # Run the p2p protocol + for _ in range(100): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + # At least one connection should be rejected during grace period (no bootstrap peers) + self.assertTrue(conn.tr1.disconnecting or conn.tr2.disconnecting) + + def test_grace_period_connections_allowed_with_bootstrap(self) -> None: + """Test that connections are allowed during grace period when peers are registered as bootstrap.""" + network = 'testnet' + manager1 = self.create_peer(network, url_whitelist='https://whitelist1.com') + manager2 = self.create_peer(network, url_whitelist='https://whitelist2.com') + + # Verify both whitelists have not had successful fetches + self.assertFalse(manager1.connections.peers_whitelist._has_successful_fetch) + self.assertFalse(manager2.connections.peers_whitelist._has_successful_fetch) + + # Register each peer as a bootstrap peer on the other's whitelist + manager1.connections.peers_whitelist.add_bootstrap_peer(manager2.my_peer.id) + manager2.connections.peers_whitelist.add_bootstrap_peer(manager1.my_peer.id) + + conn = FakeConnection(manager1, manager2) + + # Run the p2p protocol + for _ in range(100): + conn.run_one_step(debug=True) + self.clock.advance(0.1) + + # Both connections should remain up during grace period (peers are registered as bootstrap) + self.assertFalse(conn.tr1.disconnecting) + self.assertFalse(conn.tr2.disconnecting) + + def test_grace_period_flag_persists_through_failures(self) -> None: + """Test that grace period flag is NOT set on update failures.""" + whitelist = URLPeersWhitelist(self.clock, 'https://whitelist.com', mainnet=False) + + # Initial state: no successful fetch + self.assertFalse(whitelist._has_successful_fetch) + + # Simulate failures + whitelist._on_update_failure() + self.assertFalse(whitelist._has_successful_fetch) + + whitelist._on_update_failure() + self.assertFalse(whitelist._has_successful_fetch) + + # Only successful update should set the flag + whitelist._apply_whitelist_update(set(), WhitelistPolicy.ONLY_WHITELISTED_PEERS) + self.assertTrue(whitelist._has_successful_fetch) + + def test_file_whitelist_grace_period(self) -> None: + """Test that file whitelist also has grace period behavior.""" + content = """hathor-whitelist +policy: only-whitelisted-peers +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(content) + f.flush() + path = f.name + + whitelist = FilePeersWhitelist(self.clock, path) + + # Initial state: no successful fetch + self.assertFalse(whitelist._has_successful_fetch) + + # During grace period, non-bootstrap peers should NOT be allowed + random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + + # But bootstrap peers should be allowed + bootstrap_peer_id = PeerId('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890') + whitelist.add_bootstrap_peer(bootstrap_peer_id) + self.assertTrue(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + + # Perform an update + with patch('hathor.p2p.whitelist.file_whitelist.threads.deferToThread') as mock_defer: + def call_directly(func: Any, *args: Any, **kwargs: Any) -> Deferred[None]: + d: Deferred[None] = Deferred() + try: + result = func(*args, **kwargs) + d.callback(result) + except Exception as e: + d.errback(e) + return d + mock_defer.side_effect = call_directly + whitelist.update() + + # After successful fetch, grace period ends + self.assertTrue(whitelist._has_successful_fetch) + + # Now the random peer should not be allowed + self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + + # Bootstrap peer should also NOT be allowed anymore (not in whitelist) + self.assertFalse(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + + # But the whitelisted peer should be allowed + whitelisted_peer_id = PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367') + self.assertTrue(whitelist.is_peer_whitelisted(whitelisted_peer_id)) + + +class WhitelistSpecConstantsTestCase(unittest.TestCase): + """Tests for whitelist specification constants.""" + + def test_whitelist_spec_constants_values(self) -> None: + """Test that whitelist spec constants have expected values.""" + self.assertEqual(WHITELIST_SPEC_DEFAULT, 'default') + self.assertEqual(WHITELIST_SPEC_HATHORLABS, 'hathorlabs') + self.assertEqual(WHITELIST_SPEC_NONE, 'none') + self.assertEqual(WHITELIST_SPEC_DISABLED, 'disabled') diff --git a/hathor_tests/resources/p2p/test_status.py b/hathor_tests/resources/p2p/test_status.py index a01b9a7db..50bbc3df0 100644 --- a/hathor_tests/resources/p2p/test_status.py +++ b/hathor_tests/resources/p2p/test_status.py @@ -5,6 +5,7 @@ import hathor from hathor.p2p.peer_endpoint import PeerAddress from hathor.p2p.resources import StatusResource +from hathor.p2p.whitelist import URLPeersWhitelist from hathor.simulator import FakeConnection from hathor_tests.resources.base_resource import StubSite, _BaseResourceTest @@ -15,12 +16,30 @@ def setUp(self): self.web = StubSite(StatusResource(self.manager)) address1 = IPv4Address('TCP', '192.168.1.1', 54321) self.manager.connections.my_peer.info.entrypoints.add(PeerAddress.from_address(address1)) - self.manager.peers_whitelist.append(self.get_random_peer_from_pool().id) - self.manager.peers_whitelist.append(self.get_random_peer_from_pool().id) - - self.manager2 = self.create_peer('testnet') + url = "https://anything.com" + reactor = self.manager.reactor + mock_peers_whitelist = URLPeersWhitelist(reactor, url, True) + mock_peers_whitelist.start(mock_peers_whitelist._on_remove_callback) + self.manager.connections.peers_whitelist = mock_peers_whitelist + self.manager.connections.peers_whitelist.add_peer(self.get_random_peer_from_pool().id) + self.manager.connections.peers_whitelist.add_peer(self.get_random_peer_from_pool().id) + # Simulate successful fetch to end grace period + self.manager.connections.peers_whitelist._has_successful_fetch = True + url_2 = "https://somethingDifferent.com" + self.manager2 = self.create_peer('testnet', url_whitelist=url_2) address2 = IPv4Address('TCP', '192.168.1.1', 54322) self.manager2.connections.my_peer.info.entrypoints.add(PeerAddress.from_address(address2)) + + # Manager's whitelist is not empty, so its mock whitelist will be followed. + # Since manager 2 is a different instance, we need to add it to the whitelist of manager 1 + self.manager.connections.peers_whitelist.add_peer(self.manager2.my_peer.id) + + # Likewise for manager 1 in manager 2 + self.manager2.connections.peers_whitelist.add_peer(self.manager.my_peer.id) + # Simulate successful fetch to end grace period + self.manager2.connections.peers_whitelist._has_successful_fetch = True + + # Now, we create a fake connection between the two managers. self.conn1 = FakeConnection(self.manager, self.manager2, addr1=address1, addr2=address2) @inlineCallbacks @@ -79,6 +98,9 @@ def test_get_with_one_peer(self): self.conn1.run_one_step() # READY self.conn1.run_one_step() # BOTH PEERS ARE READY NOW + assert self.manager.connections.peers_whitelist is not None, 'Peers whitelist should not be None' + assert len(self.manager.connections.peers_whitelist._current) == 3, 'Should have one peer in the whitelist' + response = yield self.web.get("status") data = response.json_value() server_data = data.get('server') diff --git a/hathor_tests/unittest.py b/hathor_tests/unittest.py index bbcde3d48..835bedc36 100644 --- a/hathor_tests/unittest.py +++ b/hathor_tests/unittest.py @@ -205,6 +205,7 @@ def create_peer( # type: ignore[no-untyped-def] enable_event_queue: bool | None = None, enable_ipv6: bool = False, disable_ipv4: bool = False, + url_whitelist: str = '', nc_indexes: bool = False, nc_log_config: NCLogConfig | None = None, settings: HathorSettings | None = None, @@ -261,6 +262,9 @@ def create_peer( # type: ignore[no-untyped-def] daa = DifficultyAdjustmentAlgorithm(settings=self._settings, test_mode=TestMode.TEST_ALL_WEIGHT) builder.set_daa(daa) + if url_whitelist: + builder.set_url_whitelist(self.reactor, url=url_whitelist) + if nc_indexes: builder.enable_nc_indexes() diff --git a/hathor_tests/utils.py b/hathor_tests/utils.py index 4866987ae..251dd67f3 100644 --- a/hathor_tests/utils.py +++ b/hathor_tests/utils.py @@ -282,7 +282,9 @@ def run_server( '--status {}'.format(status), # We must allow mining without peers, otherwise some tests won't be able to mine. '--allow-mining-without-peers', - '--wallet-index' + '--wallet-index', + # Disable whitelist for testing (empty whitelist with restrictive policy blocks all) + '--x-p2p-whitelist disabled' ]) if bootstrap: diff --git a/hathorlib/hathorlib/conf/mainnet.yml b/hathorlib/hathorlib/conf/mainnet.yml index 6f1080b1e..3cf338c36 100644 --- a/hathorlib/hathorlib/conf/mainnet.yml +++ b/hathorlib/hathorlib/conf/mainnet.yml @@ -3,7 +3,6 @@ MULTISIG_VERSION_BYTE: x64 NETWORK_NAME: mainnet BOOTSTRAP_DNS: - mainnet.hathor.network -ENABLE_PEER_WHITELIST: true WHITELIST_URL: https://hathor-public-files.s3.amazonaws.com/whitelist_peer_ids # Genesis stuff From 8eab4e9e9e0f0bf32c4b170ab005f523f6230724 Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Fri, 20 Feb 2026 10:46:27 -0600 Subject: [PATCH 2/4] review changes --- hathor/p2p/manager.py | 5 +- hathor/p2p/states/peer_id.py | 2 +- hathor/p2p/whitelist/factory.py | 4 +- hathor/p2p/whitelist/parsing.py | 53 ++++++++------ hathor/p2p/whitelist/peers_whitelist.py | 18 ++--- hathor/p2p/whitelist/url_whitelist.py | 2 +- hathor/sysctl/p2p/manager.py | 20 +++--- hathor_cli/run_node.py | 2 - hathor_cli/run_node_args.py | 1 - hathor_tests/others/test_cli_builder.py | 9 +-- hathor_tests/p2p/test_whitelist.py | 96 +++++++++++++++---------- 11 files changed, 120 insertions(+), 92 deletions(-) diff --git a/hathor/p2p/manager.py b/hathor/p2p/manager.py index f147b276f..c01acdbb9 100644 --- a/hathor/p2p/manager.py +++ b/hathor/p2p/manager.py @@ -892,7 +892,8 @@ def set_peers_whitelist(self, whitelist: PeersWhitelist | None) -> None: def _disconnect_non_whitelisted_peers(self) -> None: """Disconnect all connected peers that are not in the current whitelist.""" - if not self.peers_whitelist: + whitelist = self.peers_whitelist + if not whitelist: return self.log.info('Whitelist ON: disconnecting non-whitelisted peers...') connections_snapshot = list(self.connections) @@ -900,5 +901,5 @@ def _disconnect_non_whitelisted_peers(self) -> None: peer_id = conn.get_peer_id() if peer_id is None: continue - if not self.peers_whitelist.is_peer_whitelisted(peer_id): + if not whitelist.is_peer_allowed(peer_id): conn.disconnect(reason='Whitelist updated', force=True) diff --git a/hathor/p2p/states/peer_id.py b/hathor/p2p/states/peer_id.py index 66edde341..cc7c897de 100644 --- a/hathor/p2p/states/peer_id.py +++ b/hathor/p2p/states/peer_id.py @@ -166,4 +166,4 @@ def _is_peer_allowed(self, peer_id: PeerId) -> bool: peers_whitelist = self.protocol.connections.peers_whitelist if peers_whitelist is None: return True - return peers_whitelist.is_peer_whitelisted(peer_id) + return peers_whitelist.is_peer_allowed(peer_id) diff --git a/hathor/p2p/whitelist/factory.py b/hathor/p2p/whitelist/factory.py index 5e9ee9825..640b4a851 100644 --- a/hathor/p2p/whitelist/factory.py +++ b/hathor/p2p/whitelist/factory.py @@ -47,7 +47,7 @@ def create_peers_whitelist( PeersWhitelist instance or None if disabled """ peers_whitelist: PeersWhitelist | None = None - spec_lower = whitelist_spec.lower() + spec_lower = whitelist_spec.lower().strip() if spec_lower in (WHITELIST_SPEC_DEFAULT, WHITELIST_SPEC_HATHORLABS): peers_whitelist = URLPeersWhitelist(reactor, str(settings.WHITELIST_URL), True) @@ -55,6 +55,8 @@ def create_peers_whitelist( peers_whitelist = None elif os.path.isfile(whitelist_spec): peers_whitelist = FilePeersWhitelist(reactor, whitelist_spec) + elif whitelist_spec.startswith('/') or whitelist_spec.startswith('.'): + raise ValueError(f'whitelist file not found: {whitelist_spec}') else: # URLPeersWhitelist class rejects non-url paths. peers_whitelist = URLPeersWhitelist(reactor, whitelist_spec, True) diff --git a/hathor/p2p/whitelist/parsing.py b/hathor/p2p/whitelist/parsing.py index eb62c08bc..71c903b32 100644 --- a/hathor/p2p/whitelist/parsing.py +++ b/hathor/p2p/whitelist/parsing.py @@ -59,7 +59,11 @@ def parse_whitelist_with_policy( """Parses the whitelist file and extracts both peer IDs and policy. The policy line (optional) must appear in the header, before any peer IDs. - Format: policy: + Format: # policy: + + Both ``# policy:`` and ``#policy:`` (with or without space) are accepted. + The comment-style prefix ensures backwards compatibility — older parsers + will skip the line as a comment. Policy types: - allow-all: Allow connections from any peer @@ -68,13 +72,13 @@ def parse_whitelist_with_policy( Example: parse_whitelist_with_policy('''hathor-whitelist -policy: allow-all +# policy: allow-all ''') (set(), WhitelistPolicy.ALLOW_ALL) parse_whitelist_with_policy('''hathor-whitelist # This whitelist only allows specific peers -policy: only-whitelisted-peers +# policy: only-whitelisted-peers 2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 ''') ({'2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'}, WhitelistPolicy.ONLY_WHITELISTED_PEERS) @@ -84,35 +88,44 @@ def parse_whitelist_with_policy( ''') ({'2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'}, WhitelistPolicy.ONLY_WHITELISTED_PEERS) """ - lines = parse_file(text, header=header) - policy = WhitelistPolicy.ONLY_WHITELISTED_PEERS # default + if header is None: + header = 'hathor-whitelist' + lines = text.splitlines() + _header = lines.pop(0) + if _header != header: + raise ValueError('invalid header') + policy = WhitelistPolicy.ONLY_WHITELISTED_PEERS # default peer_lines: list[str] = [] - for line in lines: - if line.startswith('policy:'): - if len(peer_lines) > 0: - raise ValueError('policy must be defined in the header, before any peer IDs') - policy_value = line.split(':', 1)[1].strip().lower() - try: - policy = WhitelistPolicy(policy_value) - except ValueError: - raise ValueError(f'invalid whitelist policy: {policy_value}') + for raw_line in lines: + line = raw_line.strip() + if not line: + continue + if line.startswith('#'): + comment_body = line.lstrip('#').lstrip() + if comment_body.startswith('policy:'): + if len(peer_lines) > 0: + raise ValueError('policy must be defined in the header, before any peer IDs') + policy_value = comment_body.split(':', 1)[1].strip().lower() + try: + policy = WhitelistPolicy(policy_value) + except ValueError: + logger.warning('invalid whitelist policy, using default', policy_value=policy_value) + continue else: peer_lines.append(line) - peers = {p for line in peer_lines if (p := _safe_parse_peer_id(line)) is not None} + peers = {p for line in peer_lines if (p := _parse_peer_id_lossy(line)) is not None} return peers, policy -def _safe_parse_peer_id(line: str) -> PeerId | None: - """Safely parse a peer ID from a whitelist line. +def _parse_peer_id_lossy(line: str) -> PeerId | None: + """Parse a peer ID from a whitelist line, returning None on failure. Returns the PeerId if valid, or None if parsing fails. - Logs a warning for invalid peer IDs. """ try: return PeerId(line.split()[0]) - except (ValueError, IndexError) as e: - logger.warning('Invalid peer ID in whitelist', line=line, error=str(e)) + except (ValueError, IndexError): return None diff --git a/hathor/p2p/whitelist/peers_whitelist.py b/hathor/p2p/whitelist/peers_whitelist.py index 197ffcdbf..df1a57f49 100644 --- a/hathor/p2p/whitelist/peers_whitelist.py +++ b/hathor/p2p/whitelist/peers_whitelist.py @@ -43,7 +43,7 @@ def __init__(self, reactor: Reactor) -> None: self._current: set[PeerId] = set() self._policy: WhitelistPolicy = WhitelistPolicy.ONLY_WHITELISTED_PEERS self._on_remove_callback: OnRemoveCallbackType = None - self._is_running: bool = False + self._is_updating: bool = False self._consecutive_failures: int = 0 self._has_successful_fetch: bool = False self._bootstrap_peers: set[PeerId] = set() @@ -83,17 +83,17 @@ def _on_update_failure(self) -> None: """Called when whitelist update fails. Increments backoff counter.""" self._consecutive_failures += 1 - def _handle_refresh_err(self, *args: Any, **kwargs: Any) -> None: + def _handle_refresh_err(self, failure: Any) -> None: """This method will be called when an exception happens inside the whitelist update and ends up stopping the looping call. We log the error and start the looping call again with exponential backoff. """ self._on_update_failure() retry_interval = self._get_retry_interval() - self.log.error( + self.log.warning( 'whitelist refresh had an exception. Start looping call again.', - args=args, - kwargs=kwargs, + error=failure.getErrorMessage(), + traceback=failure.getTraceback(), retry_interval=retry_interval, consecutive_failures=self._consecutive_failures ) @@ -101,15 +101,15 @@ def _handle_refresh_err(self, *args: Any, **kwargs: Any) -> None: def update(self) -> Deferred[None]: # Avoiding re-entrancy. If running, should not update once more. - if self._is_running: + if self._is_updating: self.log.warning('whitelist update already running, skipping execution.') d: Deferred[None] = Deferred() d.callback(None) return d - self._is_running = True + self._is_updating = True d = self._unsafe_update() - d.addBoth(lambda _: setattr(self, '_is_running', False)) + d.addBoth(lambda _: setattr(self, '_is_updating', False)) return d def add_peer(self, peer_id: PeerId) -> None: @@ -126,7 +126,7 @@ def policy(self) -> WhitelistPolicy: """ Returns the current whitelist policy.""" return self._policy - def is_peer_whitelisted(self, peer_id: PeerId) -> bool: + def is_peer_allowed(self, peer_id: PeerId) -> bool: """ Returns True if peer is whitelisted or policy is ALLOW_ALL. During the grace period (before first successful fetch), only bootstrap peers diff --git a/hathor/p2p/whitelist/url_whitelist.py b/hathor/p2p/whitelist/url_whitelist.py index 85f04cb73..a1c5e883e 100644 --- a/hathor/p2p/whitelist/url_whitelist.py +++ b/hathor/p2p/whitelist/url_whitelist.py @@ -34,7 +34,7 @@ def __init__(self, reactor: Reactor, url: str | None, mainnet: bool = False) -> if self._url is None: return - if self._url.lower() == 'none': + if self._url.lower().strip() == 'none': self._url = None return diff --git a/hathor/sysctl/p2p/manager.py b/hathor/sysctl/p2p/manager.py index ab3b3a96c..09afac98b 100644 --- a/hathor/sysctl/p2p/manager.py +++ b/hathor/sysctl/p2p/manager.py @@ -319,37 +319,39 @@ def set_whitelist(self, new_whitelist: str) -> None: the whitelist object, following it by default. It does not support eliminating the whitelist (passing None).""" + connections = self.connections option: str = new_whitelist.lower().strip() if option == 'on': if self._suspended_whitelist is None: return - self.connections.set_peers_whitelist(self._suspended_whitelist) + connections.set_peers_whitelist(self._suspended_whitelist) self._suspended_whitelist = None return if option == 'off': - if self.connections.peers_whitelist is None: + if connections.peers_whitelist is None: return - self._suspended_whitelist = self.connections.peers_whitelist - self.connections.set_peers_whitelist(None) + self._suspended_whitelist = connections.peers_whitelist + connections.set_peers_whitelist(None) return from hathor.p2p.whitelist import create_peers_whitelist whitelist = create_peers_whitelist( - self.connections.reactor, + connections.reactor, new_whitelist, - self.connections._settings, + connections._settings, ) if whitelist is None: raise SysctlException('Sysctl does not allow whitelist swap to None. Use "off" to disable it.') self._suspended_whitelist = None - self.connections.set_peers_whitelist(whitelist) + connections.set_peers_whitelist(whitelist) def whitelist_status(self) -> WhitelistStatus: """Return structured status information about the whitelist.""" - if self.connections.peers_whitelist is not None: - whitelist = self.connections.peers_whitelist + connections = self.connections + if connections.peers_whitelist is not None: + whitelist = connections.peers_whitelist return WhitelistStatus( state=WhitelistState.ON, policy=whitelist.policy(), diff --git a/hathor_cli/run_node.py b/hathor_cli/run_node.py index 9d3c82d19..564551cb8 100644 --- a/hathor_cli/run_node.py +++ b/hathor_cli/run_node.py @@ -175,8 +175,6 @@ def create_parser(cls) -> ArgumentParser: parser.add_argument('--x-disable-ipv4', action='store_true', help='Disables connecting to IPv4 peers') - parser.add_argument("--x-p2p-whitelist-only", action="store_true", - help="Node will only connect to peers on the whitelist") parser.add_argument("--x-p2p-whitelist", help="Add whitelist to follow from since boot.") possible_nc_exec_logs = [config.value for config in NCLogConfig] diff --git a/hathor_cli/run_node_args.py b/hathor_cli/run_node_args.py index 8486cbdf8..d77c10897 100644 --- a/hathor_cli/run_node_args.py +++ b/hathor_cli/run_node_args.py @@ -93,7 +93,6 @@ class RunNodeArgs(BaseModel): x_enable_ipv6: bool x_disable_ipv4: bool localnet: bool - x_p2p_whitelist_only: bool x_p2p_whitelist: Optional[str] nc_indexes: bool nc_exec_logs: NCLogConfig diff --git a/hathor_tests/others/test_cli_builder.py b/hathor_tests/others/test_cli_builder.py index a2e4043e6..9d3b5e4c7 100644 --- a/hathor_tests/others/test_cli_builder.py +++ b/hathor_tests/others/test_cli_builder.py @@ -137,19 +137,12 @@ def test_event_queue_with_rocksdb_storage(self): self.assertTrue(manager._enable_event_queue) def test_whitelist_cli_args(self): - """Test --x-p2p-whitelist and --x-p2p-whitelist-only CLI arguments.""" + """Test --x-p2p-whitelist CLI argument.""" # Test with whitelist URL manager = self._build(['--temp-data', '--x-p2p-whitelist', 'https://example.com/whitelist']) self.assertIsNotNone(manager.connections.peers_whitelist) self.assertIsInstance(manager.connections.peers_whitelist, URLPeersWhitelist) - # Test with whitelist-only flag (now a no-op, whitelist always enforces) - manager2 = self._build([ - '--temp-data', '--x-p2p-whitelist', 'https://example.com/whitelist', - '--x-p2p-whitelist-only', - ]) - self.assertIsNotNone(manager2.connections.peers_whitelist) - # Test with disabled whitelist manager3 = self._build(['--temp-data', '--x-p2p-whitelist', 'none']) self.assertIsNone(manager3.connections.peers_whitelist) diff --git a/hathor_tests/p2p/test_whitelist.py b/hathor_tests/p2p/test_whitelist.py index 1c2fc68c7..2b6f99ecf 100644 --- a/hathor_tests/p2p/test_whitelist.py +++ b/hathor_tests/p2p/test_whitelist.py @@ -241,7 +241,7 @@ class ParseWhitelistWithPolicyTestCase(unittest.TestCase): def test_parse_allow_all_policy(self) -> None: """Test parsing whitelist with allow-all policy.""" content = """hathor-whitelist -policy: allow-all +#policy: allow-all """ peers, policy = parse_whitelist_with_policy(content) self.assertEqual(policy, WhitelistPolicy.ALLOW_ALL) @@ -250,7 +250,7 @@ def test_parse_allow_all_policy(self) -> None: def test_parse_only_whitelisted_peers_policy(self) -> None: """Test parsing whitelist with only-whitelisted-peers policy.""" content = """hathor-whitelist -policy: only-whitelisted-peers +#policy: only-whitelisted-peers 2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 """ peers, policy = parse_whitelist_with_policy(content) @@ -271,7 +271,7 @@ def test_parse_policy_with_comments(self) -> None: """Test parsing whitelist with policy and comments.""" content = """hathor-whitelist # This whitelist allows all peers -policy: allow-all +#policy: allow-all # More comments here """ peers, policy = parse_whitelist_with_policy(content) @@ -282,25 +282,45 @@ def test_parse_policy_after_peer_raises_error(self) -> None: """Test that policy line after peer IDs raises ValueError.""" content = """hathor-whitelist 2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 -policy: allow-all +#policy: allow-all """ with self.assertRaises(ValueError) as context: parse_whitelist_with_policy(content) self.assertIn('policy must be defined in the header', str(context.exception)) - def test_parse_invalid_policy(self) -> None: - """Test that invalid policy raises ValueError.""" + def test_parse_invalid_policy_uses_default(self) -> None: + """Test that invalid policy logs a warning and uses the default.""" content = """hathor-whitelist -policy: invalid-policy +#policy: invalid-policy """ - with self.assertRaises(ValueError) as context: - parse_whitelist_with_policy(content) - self.assertIn('invalid whitelist policy', str(context.exception)) + peers, policy = parse_whitelist_with_policy(content) + self.assertEqual(policy, WhitelistPolicy.ONLY_WHITELISTED_PEERS) + self.assertEqual(peers, set()) + + def test_parse_policy_with_space_after_hash(self) -> None: + """Test parsing whitelist with '# policy:' (space after #).""" + content = """hathor-whitelist +# policy: allow-all +2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 +""" + peers, policy = parse_whitelist_with_policy(content) + self.assertEqual(policy, WhitelistPolicy.ALLOW_ALL) + self.assertEqual(len(peers), 1) + self.assertIn(PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367'), peers) + + def test_parse_policy_no_space_after_hash(self) -> None: + """Test parsing whitelist with '#policy:' (no space after #) still works.""" + content = """hathor-whitelist +#policy: allow-all +""" + peers, policy = parse_whitelist_with_policy(content) + self.assertEqual(policy, WhitelistPolicy.ALLOW_ALL) + self.assertEqual(peers, set()) def test_parse_policy_case_insensitive(self) -> None: """Test that policy values are case-insensitive.""" content = """hathor-whitelist -policy: ALLOW-ALL +#policy: ALLOW-ALL """ peers, policy = parse_whitelist_with_policy(content) self.assertEqual(policy, WhitelistPolicy.ALLOW_ALL) @@ -309,8 +329,8 @@ def test_parse_policy_case_insensitive(self) -> None: class WhitelistPolicyBehaviorTestCase(unittest.TestCase): """Tests for WhitelistPolicy behavior in PeersWhitelist.""" - def test_is_peer_whitelisted_with_allow_all(self) -> None: - """Test that is_peer_whitelisted returns True with ALLOW_ALL policy after grace period.""" + def test_is_peer_allowed_with_allow_all(self) -> None: + """Test that is_peer_allowed returns True with ALLOW_ALL policy after grace period.""" network = 'testnet' manager = self.create_peer(network, url_whitelist='https://whitelist.com') whitelist = manager.connections.peers_whitelist @@ -323,10 +343,10 @@ def test_is_peer_whitelisted_with_allow_all(self) -> None: # Even with an empty whitelist, any peer should be allowed with ALLOW_ALL policy random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') - self.assertTrue(whitelist.is_peer_whitelisted(random_peer_id)) + self.assertTrue(whitelist.is_peer_allowed(random_peer_id)) - def test_is_peer_whitelisted_with_only_whitelisted_peers(self) -> None: - """Test that is_peer_whitelisted checks whitelist with ONLY_WHITELISTED_PEERS policy after grace period.""" + def test_is_peer_allowed_with_only_whitelisted_peers(self) -> None: + """Test that is_peer_allowed checks whitelist with ONLY_WHITELISTED_PEERS policy after grace period.""" network = 'testnet' manager = self.create_peer(network, url_whitelist='https://whitelist.com') whitelist = manager.connections.peers_whitelist @@ -342,11 +362,11 @@ def test_is_peer_whitelisted_with_only_whitelisted_peers(self) -> None: whitelist.add_peer(whitelisted_peer_id) # Whitelisted peer should be allowed - self.assertTrue(whitelist.is_peer_whitelisted(whitelisted_peer_id)) + self.assertTrue(whitelist.is_peer_allowed(whitelisted_peer_id)) # Non-whitelisted peer should not be allowed after grace period random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') - self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) def test_policy_method(self) -> None: """Test the policy() method returns correct policy.""" @@ -368,7 +388,7 @@ class FilePeersWhitelistTestCase(unittest.TestCase): def test_file_whitelist_reads_valid_file(self) -> None: """Test that FilePeersWhitelist correctly reads a valid whitelist file.""" content = """hathor-whitelist -policy: only-whitelisted-peers +#policy: only-whitelisted-peers 2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef """ @@ -448,7 +468,7 @@ def raise_permission_error(func: Any, *args: Any, **kwargs: Any) -> Deferred[Non def test_file_whitelist_refresh_returns_deferred(self) -> None: """Test that FilePeersWhitelist.refresh() returns a Deferred.""" content = """hathor-whitelist -policy: allow-all +#policy: allow-all """ with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: f.write(content) @@ -622,7 +642,7 @@ def test_sysctl_swap_to_file(self) -> None: # Create a temporary file whitelist content = """hathor-whitelist -policy: allow-all +#policy: allow-all """ with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: f.write(content) @@ -754,7 +774,7 @@ def test_empty_whitelist_with_only_whitelisted_peers_blocks_all(self) -> None: # Any peer should NOT be whitelisted with empty list + restrictive policy after grace period random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') - self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) def test_empty_whitelist_with_allow_all_allows_all(self) -> None: """Test that empty whitelist with ALLOW_ALL policy allows all peers after grace period.""" @@ -773,7 +793,7 @@ def test_empty_whitelist_with_allow_all_allows_all(self) -> None: # Any peer should be whitelisted with ALLOW_ALL policy random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') - self.assertTrue(whitelist.is_peer_whitelisted(random_peer_id)) + self.assertTrue(whitelist.is_peer_allowed(random_peer_id)) def test_empty_whitelist_blocks_connection_with_restrictive_policy(self) -> None: """Test that empty whitelist blocks connections when policy is ONLY_WHITELISTED_PEERS after grace period.""" @@ -954,7 +974,7 @@ def test_whitelist_source_method(self) -> None: # Test FilePeersWhitelist with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: - f.write("hathor-whitelist\npolicy: allow-all\n") + f.write("hathor-whitelist\n#policy: allow-all\n") f.flush() path = f.name @@ -978,7 +998,7 @@ def test_grace_period_rejects_non_bootstrap_peers_before_first_fetch(self) -> No # Random peers should NOT be allowed during grace period (only bootstrap peers allowed) random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') - self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) def test_grace_period_allows_bootstrap_peers_before_first_fetch(self) -> None: """Test that bootstrap peers are allowed before first successful whitelist fetch.""" @@ -996,11 +1016,11 @@ def test_grace_period_allows_bootstrap_peers_before_first_fetch(self) -> None: whitelist.add_bootstrap_peer(bootstrap_peer_id) # Bootstrap peer should be allowed during grace period - self.assertTrue(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + self.assertTrue(whitelist.is_peer_allowed(bootstrap_peer_id)) # Non-bootstrap peer should still be rejected random_peer_id = PeerId('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890') - self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) def test_grace_period_ends_after_successful_fetch(self) -> None: """Test that grace period ends after first successful whitelist fetch.""" @@ -1019,11 +1039,11 @@ def test_grace_period_ends_after_successful_fetch(self) -> None: self.assertTrue(whitelist._has_successful_fetch) # Now the whitelist should enforce - whitelisted peer is allowed - self.assertTrue(whitelist.is_peer_whitelisted(whitelisted_peer_id)) + self.assertTrue(whitelist.is_peer_allowed(whitelisted_peer_id)) # Non-whitelisted peer should be blocked after grace period random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') - self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) def test_bootstrap_peer_allowed_during_grace_period_but_normal_rules_after(self) -> None: """Test that bootstrap peers follow normal whitelist rules after successful fetch.""" @@ -1039,7 +1059,7 @@ def test_bootstrap_peer_allowed_during_grace_period_but_normal_rules_after(self) whitelist.add_bootstrap_peer(bootstrap_peer_id) # Bootstrap peer should be allowed during grace period - self.assertTrue(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + self.assertTrue(whitelist.is_peer_allowed(bootstrap_peer_id)) # Simulate a successful whitelist update that does NOT include the bootstrap peer other_peer_id = PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367') @@ -1049,10 +1069,10 @@ def test_bootstrap_peer_allowed_during_grace_period_but_normal_rules_after(self) self.assertTrue(whitelist._has_successful_fetch) # Bootstrap peer should NOT be allowed anymore (not in whitelist) - self.assertFalse(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + self.assertFalse(whitelist.is_peer_allowed(bootstrap_peer_id)) # Peer in whitelist should be allowed - self.assertTrue(whitelist.is_peer_whitelisted(other_peer_id)) + self.assertTrue(whitelist.is_peer_allowed(other_peer_id)) def test_grace_period_connections_rejected_without_bootstrap(self) -> None: """Test that connections are rejected during grace period without bootstrap registration.""" @@ -1124,7 +1144,7 @@ def test_grace_period_flag_persists_through_failures(self) -> None: def test_file_whitelist_grace_period(self) -> None: """Test that file whitelist also has grace period behavior.""" content = """hathor-whitelist -policy: only-whitelisted-peers +#policy: only-whitelisted-peers 2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 """ with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: @@ -1139,12 +1159,12 @@ def test_file_whitelist_grace_period(self) -> None: # During grace period, non-bootstrap peers should NOT be allowed random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') - self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) # But bootstrap peers should be allowed bootstrap_peer_id = PeerId('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890') whitelist.add_bootstrap_peer(bootstrap_peer_id) - self.assertTrue(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + self.assertTrue(whitelist.is_peer_allowed(bootstrap_peer_id)) # Perform an update with patch('hathor.p2p.whitelist.file_whitelist.threads.deferToThread') as mock_defer: @@ -1163,14 +1183,14 @@ def call_directly(func: Any, *args: Any, **kwargs: Any) -> Deferred[None]: self.assertTrue(whitelist._has_successful_fetch) # Now the random peer should not be allowed - self.assertFalse(whitelist.is_peer_whitelisted(random_peer_id)) + self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) # Bootstrap peer should also NOT be allowed anymore (not in whitelist) - self.assertFalse(whitelist.is_peer_whitelisted(bootstrap_peer_id)) + self.assertFalse(whitelist.is_peer_allowed(bootstrap_peer_id)) # But the whitelisted peer should be allowed whitelisted_peer_id = PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367') - self.assertTrue(whitelist.is_peer_whitelisted(whitelisted_peer_id)) + self.assertTrue(whitelist.is_peer_allowed(whitelisted_peer_id)) class WhitelistSpecConstantsTestCase(unittest.TestCase): From bbbf1160d166df03f2e70d1cb35c31b477f1c678 Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Fri, 20 Feb 2026 15:11:55 -0600 Subject: [PATCH 3/4] review changes (2) --- hathor/p2p/whitelist/factory.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/hathor/p2p/whitelist/factory.py b/hathor/p2p/whitelist/factory.py index 640b4a851..f1bbdcdfd 100644 --- a/hathor/p2p/whitelist/factory.py +++ b/hathor/p2p/whitelist/factory.py @@ -14,6 +14,7 @@ import os from typing import TYPE_CHECKING +from urllib.parse import urlparse from hathor.p2p.whitelist.file_whitelist import FilePeersWhitelist from hathor.p2p.whitelist.peers_whitelist import PeersWhitelist @@ -30,6 +31,11 @@ WHITELIST_SPEC_DISABLED = 'disabled' +def _looks_like_url(spec: str) -> bool: + parsed = urlparse(spec) + return parsed.scheme in ('http', 'https') + + def create_peers_whitelist( reactor: Reactor, whitelist_spec: str, @@ -53,12 +59,11 @@ def create_peers_whitelist( peers_whitelist = URLPeersWhitelist(reactor, str(settings.WHITELIST_URL), True) elif spec_lower in (WHITELIST_SPEC_NONE, WHITELIST_SPEC_DISABLED): peers_whitelist = None + elif _looks_like_url(whitelist_spec): + peers_whitelist = URLPeersWhitelist(reactor, whitelist_spec, True) elif os.path.isfile(whitelist_spec): peers_whitelist = FilePeersWhitelist(reactor, whitelist_spec) - elif whitelist_spec.startswith('/') or whitelist_spec.startswith('.'): - raise ValueError(f'whitelist file not found: {whitelist_spec}') else: - # URLPeersWhitelist class rejects non-url paths. - peers_whitelist = URLPeersWhitelist(reactor, whitelist_spec, True) + raise ValueError(f'whitelist spec is not a URL and file does not exist: {whitelist_spec}') return peers_whitelist From 334dea1509606b259090add2faeee3c1ecf4eb13 Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Fri, 20 Feb 2026 20:07:11 -0600 Subject: [PATCH 4/4] review changes (3) --- hathor/p2p/manager.py | 26 +++- hathor/p2p/states/peer_id.py | 10 +- hathor/p2p/whitelist/peers_whitelist.py | 13 +- hathor_tests/p2p/test_whitelist.py | 155 +++++++++++++++++++----- 4 files changed, 147 insertions(+), 57 deletions(-) diff --git a/hathor/p2p/manager.py b/hathor/p2p/manager.py index c01acdbb9..4a92cb3c2 100644 --- a/hathor/p2p/manager.py +++ b/hathor/p2p/manager.py @@ -187,6 +187,9 @@ def __init__( # Whitelisted peers. self.peers_whitelist: PeersWhitelist | None = peers_whitelist + # Bootstrap peer IDs tracked independently of any whitelist object. + self._bootstrap_peer_ids: set[PeerId] = set() + # Pubsub object to publish events self.pubsub = pubsub @@ -267,8 +270,8 @@ def do_discovery(self) -> None: for peer_discovery in self.peer_discoveries: # Wrap connect_to_endpoint to register bootstrap peer IDs def connect_with_bootstrap_registration(entrypoint: PeerEndpoint) -> None: - if self.peers_whitelist and entrypoint.peer_id is not None: - self.peers_whitelist.add_bootstrap_peer(entrypoint.peer_id) + if entrypoint.peer_id is not None: + self._bootstrap_peer_ids.add(entrypoint.peer_id) self.connect_to_endpoint(entrypoint) coro = peer_discovery.discover_and_connect(connect_with_bootstrap_registration) @@ -789,6 +792,9 @@ def drop_connection(self, protocol: HathorProtocol) -> None: def drop_connection_by_peer_id(self, peer_id: PeerId) -> None: """ Drop a connection by peer id """ + if peer_id in self._bootstrap_peer_ids: + self.log.debug('skipping disconnect of bootstrap peer', peer_id=peer_id) + return protocol = self.connected_peers.get(peer_id) if protocol: self.drop_connection(protocol) @@ -890,10 +896,17 @@ def set_peers_whitelist(self, whitelist: PeersWhitelist | None) -> None: whitelist.start(self.drop_connection_by_peer_id) self._disconnect_non_whitelisted_peers() + def is_peer_allowed(self, peer_id: PeerId) -> bool: + """Return True if peer is allowed to connect; False otherwise.""" + if peer_id in self._bootstrap_peer_ids: + return True + if self.peers_whitelist is None: + return True + return self.peers_whitelist.is_peer_allowed(peer_id) + def _disconnect_non_whitelisted_peers(self) -> None: """Disconnect all connected peers that are not in the current whitelist.""" - whitelist = self.peers_whitelist - if not whitelist: + if not self.peers_whitelist: return self.log.info('Whitelist ON: disconnecting non-whitelisted peers...') connections_snapshot = list(self.connections) @@ -901,5 +914,6 @@ def _disconnect_non_whitelisted_peers(self) -> None: peer_id = conn.get_peer_id() if peer_id is None: continue - if not whitelist.is_peer_allowed(peer_id): - conn.disconnect(reason='Whitelist updated', force=True) + if not self.is_peer_allowed(peer_id): + self.log.info('Disconnecting non-whitelisted peer.', peer_id=str(peer_id)) + conn.disconnect(reason='Blocked', force=True) diff --git a/hathor/p2p/states/peer_id.py b/hathor/p2p/states/peer_id.py index cc7c897de..199123246 100644 --- a/hathor/p2p/states/peer_id.py +++ b/hathor/p2p/states/peer_id.py @@ -19,7 +19,6 @@ from hathor.conf.settings import HathorSettings from hathor.p2p.messages import ProtocolMessages from hathor.p2p.peer import PublicPeer -from hathor.p2p.peer_id import PeerId from hathor.p2p.states.base import BaseState from hathor.util import json_dumps, json_loads @@ -116,7 +115,7 @@ async def handle_peer_id(self, payload: str) -> None: return # is it on the whitelist? - if not self._is_peer_allowed(peer.id): + if not self.protocol.connections.is_peer_allowed(peer.id): if self._settings.WHITELIST_WARN_BLOCKED_PEERS: protocol.send_error_and_close_connection(f'Blocked (by {peer.id}). Get in touch with Hathor team.') else: @@ -160,10 +159,3 @@ async def handle_peer_id(self, payload: str) -> None: return self.send_ready() - - def _is_peer_allowed(self, peer_id: PeerId) -> bool: - """Return True if peer is allowed to connect; False otherwise.""" - peers_whitelist = self.protocol.connections.peers_whitelist - if peers_whitelist is None: - return True - return peers_whitelist.is_peer_allowed(peer_id) diff --git a/hathor/p2p/whitelist/peers_whitelist.py b/hathor/p2p/whitelist/peers_whitelist.py index df1a57f49..d65ad54c2 100644 --- a/hathor/p2p/whitelist/peers_whitelist.py +++ b/hathor/p2p/whitelist/peers_whitelist.py @@ -46,12 +46,6 @@ def __init__(self, reactor: Reactor) -> None: self._is_updating: bool = False self._consecutive_failures: int = 0 self._has_successful_fetch: bool = False - self._bootstrap_peers: set[PeerId] = set() - - def add_bootstrap_peer(self, peer_id: PeerId) -> None: - """Add a bootstrap peer ID. These are allowed during grace period.""" - self._bootstrap_peers.add(peer_id) - self.log.debug('Bootstrap peer added', peer_id=peer_id) def start(self, on_remove_callback: OnRemoveCallbackType) -> None: self._on_remove_callback = on_remove_callback @@ -129,12 +123,11 @@ def policy(self) -> WhitelistPolicy: def is_peer_allowed(self, peer_id: PeerId) -> bool: """ Returns True if peer is whitelisted or policy is ALLOW_ALL. - During the grace period (before first successful fetch), only bootstrap peers - are allowed to prevent connecting to arbitrary peers before the whitelist is loaded. + During the grace period (before first successful fetch), all peers are blocked + to prevent connecting to arbitrary peers before the whitelist is loaded. """ - # Grace period: only allow bootstrap peers until first successful fetch if not self._has_successful_fetch: - return peer_id in self._bootstrap_peers + return False if self._policy == WhitelistPolicy.ALLOW_ALL: return True return peer_id in self._current diff --git a/hathor_tests/p2p/test_whitelist.py b/hathor_tests/p2p/test_whitelist.py index 2b6f99ecf..26ecdf4ea 100644 --- a/hathor_tests/p2p/test_whitelist.py +++ b/hathor_tests/p2p/test_whitelist.py @@ -1001,7 +1001,11 @@ def test_grace_period_rejects_non_bootstrap_peers_before_first_fetch(self) -> No self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) def test_grace_period_allows_bootstrap_peers_before_first_fetch(self) -> None: - """Test that bootstrap peers are allowed before first successful whitelist fetch.""" + """Test that bootstrap peers are allowed before first successful whitelist fetch. + + Bootstrap exemption is now handled by the manager, not the whitelist. + The manager's _is_peer_allowed checks _bootstrap_peer_ids before consulting the whitelist. + """ network = 'testnet' manager = self.create_peer(network, url_whitelist='https://whitelist.com') whitelist = manager.connections.peers_whitelist @@ -1011,14 +1015,18 @@ def test_grace_period_allows_bootstrap_peers_before_first_fetch(self) -> None: self.assertEqual(whitelist.policy(), WhitelistPolicy.ONLY_WHITELISTED_PEERS) - # Register a bootstrap peer + # Register a bootstrap peer on the manager (not on the whitelist) bootstrap_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') - whitelist.add_bootstrap_peer(bootstrap_peer_id) + manager.connections._bootstrap_peer_ids.add(bootstrap_peer_id) - # Bootstrap peer should be allowed during grace period - self.assertTrue(whitelist.is_peer_allowed(bootstrap_peer_id)) + # Bootstrap peer is in manager's _bootstrap_peer_ids, so the manager would allow it. + # The whitelist itself does NOT know about bootstrap peers anymore. + self.assertIn(bootstrap_peer_id, manager.connections._bootstrap_peer_ids) - # Non-bootstrap peer should still be rejected + # Whitelist still rejects everyone during grace period (no bootstrap awareness) + self.assertFalse(whitelist.is_peer_allowed(bootstrap_peer_id)) + + # Non-bootstrap peer should also be rejected random_peer_id = PeerId('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890') self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) @@ -1045,21 +1053,19 @@ def test_grace_period_ends_after_successful_fetch(self) -> None: random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) - def test_bootstrap_peer_allowed_during_grace_period_but_normal_rules_after(self) -> None: - """Test that bootstrap peers follow normal whitelist rules after successful fetch.""" + def test_bootstrap_peer_always_allowed(self) -> None: + """Test that bootstrap peers are always allowed via manager's _bootstrap_peer_ids. + + Bootstrap exemption is now handled by the manager, not the whitelist. + The whitelist's is_peer_allowed does NOT know about bootstrap peers. + """ network = 'testnet' manager = self.create_peer(network, url_whitelist='https://whitelist.com') whitelist = manager.connections.peers_whitelist - # Verify initial state: no successful fetch yet - self.assertFalse(whitelist._has_successful_fetch) - - # Register a bootstrap peer + # Register a bootstrap peer on the manager bootstrap_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') - whitelist.add_bootstrap_peer(bootstrap_peer_id) - - # Bootstrap peer should be allowed during grace period - self.assertTrue(whitelist.is_peer_allowed(bootstrap_peer_id)) + manager.connections._bootstrap_peer_ids.add(bootstrap_peer_id) # Simulate a successful whitelist update that does NOT include the bootstrap peer other_peer_id = PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367') @@ -1068,9 +1074,12 @@ def test_bootstrap_peer_allowed_during_grace_period_but_normal_rules_after(self) # Verify grace period has ended self.assertTrue(whitelist._has_successful_fetch) - # Bootstrap peer should NOT be allowed anymore (not in whitelist) + # Whitelist does NOT know about bootstrap peers — it rejects non-whitelisted peers self.assertFalse(whitelist.is_peer_allowed(bootstrap_peer_id)) + # But the manager would allow it via _bootstrap_peer_ids check + self.assertIn(bootstrap_peer_id, manager.connections._bootstrap_peer_ids) + # Peer in whitelist should be allowed self.assertTrue(whitelist.is_peer_allowed(other_peer_id)) @@ -1108,9 +1117,9 @@ def test_grace_period_connections_allowed_with_bootstrap(self) -> None: self.assertFalse(manager1.connections.peers_whitelist._has_successful_fetch) self.assertFalse(manager2.connections.peers_whitelist._has_successful_fetch) - # Register each peer as a bootstrap peer on the other's whitelist - manager1.connections.peers_whitelist.add_bootstrap_peer(manager2.my_peer.id) - manager2.connections.peers_whitelist.add_bootstrap_peer(manager1.my_peer.id) + # Register each peer as a bootstrap peer on the other's manager + manager1.connections._bootstrap_peer_ids.add(manager2.my_peer.id) + manager2.connections._bootstrap_peer_ids.add(manager1.my_peer.id) conn = FakeConnection(manager1, manager2) @@ -1142,7 +1151,10 @@ def test_grace_period_flag_persists_through_failures(self) -> None: self.assertTrue(whitelist._has_successful_fetch) def test_file_whitelist_grace_period(self) -> None: - """Test that file whitelist also has grace period behavior.""" + """Test that file whitelist also has grace period behavior. + + Bootstrap exemption is now handled by the manager, not the whitelist. + """ content = """hathor-whitelist #policy: only-whitelisted-peers 2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367 @@ -1157,15 +1169,10 @@ def test_file_whitelist_grace_period(self) -> None: # Initial state: no successful fetch self.assertFalse(whitelist._has_successful_fetch) - # During grace period, non-bootstrap peers should NOT be allowed + # During grace period, all peers are blocked (whitelist has no bootstrap awareness) random_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) - # But bootstrap peers should be allowed - bootstrap_peer_id = PeerId('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890') - whitelist.add_bootstrap_peer(bootstrap_peer_id) - self.assertTrue(whitelist.is_peer_allowed(bootstrap_peer_id)) - # Perform an update with patch('hathor.p2p.whitelist.file_whitelist.threads.deferToThread') as mock_defer: def call_directly(func: Any, *args: Any, **kwargs: Any) -> Deferred[None]: @@ -1182,16 +1189,100 @@ def call_directly(func: Any, *args: Any, **kwargs: Any) -> Deferred[None]: # After successful fetch, grace period ends self.assertTrue(whitelist._has_successful_fetch) - # Now the random peer should not be allowed + # Now the random peer should not be allowed (not in whitelist) self.assertFalse(whitelist.is_peer_allowed(random_peer_id)) - # Bootstrap peer should also NOT be allowed anymore (not in whitelist) - self.assertFalse(whitelist.is_peer_allowed(bootstrap_peer_id)) - - # But the whitelisted peer should be allowed + # The whitelisted peer should be allowed whitelisted_peer_id = PeerId('2ffdfbbfd6d869a0742cff2b054af1cf364ae4298660c0e42fa8b00a66a30367') self.assertTrue(whitelist.is_peer_allowed(whitelisted_peer_id)) + def test_bootstrap_peers_survive_late_whitelist_activation(self) -> None: + """Test that bootstrap peers are preserved when whitelist is activated after they connected. + + Bootstrap peer IDs live on the manager, not the whitelist, so they naturally + survive whitelist swaps without any transfer logic. + """ + network = 'testnet' + # Start without a whitelist + manager = self.create_peer(network, url_whitelist='') + self.assertIsNone(manager.connections.peers_whitelist) + + # Simulate bootstrap peers connecting via do_discovery (manually register) + bootstrap_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + manager.connections._bootstrap_peer_ids.add(bootstrap_peer_id) + + # Later, activate a whitelist via sysctl (simulated with set_peers_whitelist) + new_whitelist = URLPeersWhitelist(self.clock, 'https://whitelist.com') + manager.connections.set_peers_whitelist(new_whitelist) + + # Bootstrap peer IDs remain on the manager (not on the whitelist) + self.assertIn(bootstrap_peer_id, manager.connections._bootstrap_peer_ids) + + # Non-bootstrap peer should NOT be allowed during grace period + random_peer_id = PeerId('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890') + self.assertFalse(new_whitelist.is_peer_allowed(random_peer_id)) + + def test_drop_connection_by_peer_id_skips_bootstrap_peers(self) -> None: + """Test that drop_connection_by_peer_id does not disconnect bootstrap peers.""" + network = 'testnet' + manager = self.create_peer(network, url_whitelist='') + + # Register a bootstrap peer + bootstrap_peer_id = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + manager.connections._bootstrap_peer_ids.add(bootstrap_peer_id) + + # Create a mock protocol for the bootstrap peer + mock_protocol = Mock() + mock_protocol.peer = Mock() + mock_protocol.peer.id = bootstrap_peer_id + manager.connections.connected_peers[bootstrap_peer_id] = mock_protocol + + # Attempt to drop the bootstrap peer connection + manager.connections.drop_connection_by_peer_id(bootstrap_peer_id) + + # The connection should NOT have been dropped + mock_protocol.send_error_and_close_connection.assert_not_called() + self.assertIn(bootstrap_peer_id, manager.connections.connected_peers) + + # But dropping a non-bootstrap peer should work + other_peer_id = PeerId('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890') + other_protocol = Mock() + other_protocol.peer = Mock() + other_protocol.peer.id = other_peer_id + manager.connections.connected_peers[other_peer_id] = other_protocol + + manager.connections.drop_connection_by_peer_id(other_peer_id) + other_protocol.send_error_and_close_connection.assert_called_once() + + def test_whitelist_swap_preserves_bootstrap_peer_ids_on_manager(self) -> None: + """Test that swapping whitelists preserves bootstrap peer IDs on the manager. + + Bootstrap peer IDs live on the manager, not the whitelist, so they + naturally survive whitelist swaps. + """ + network = 'testnet' + manager = self.create_peer(network, url_whitelist='https://whitelist1.com') + + # Register bootstrap peers on the manager + bootstrap_peer_id_1 = PeerId('1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') + bootstrap_peer_id_2 = PeerId('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890') + manager.connections._bootstrap_peer_ids.add(bootstrap_peer_id_1) + manager.connections._bootstrap_peer_ids.add(bootstrap_peer_id_2) + + old_whitelist = manager.connections.peers_whitelist + self.assertIsNotNone(old_whitelist) + + # Swap to a new whitelist + new_whitelist = URLPeersWhitelist(self.clock, 'https://whitelist2.com') + manager.connections.set_peers_whitelist(new_whitelist) + + # Both bootstrap peers should still be on the manager + self.assertIn(bootstrap_peer_id_1, manager.connections._bootstrap_peer_ids) + self.assertIn(bootstrap_peer_id_2, manager.connections._bootstrap_peer_ids) + + # Old whitelist should have been stopped + self.assertFalse(old_whitelist.lc_refresh.running) + class WhitelistSpecConstantsTestCase(unittest.TestCase): """Tests for whitelist specification constants."""