diff --git a/packages/rs-dpp/src/lib.rs b/packages/rs-dpp/src/lib.rs index 7a7c90a8080..a8ebf21b25b 100644 --- a/packages/rs-dpp/src/lib.rs +++ b/packages/rs-dpp/src/lib.rs @@ -61,6 +61,8 @@ pub mod core_subsidy; pub mod fee; pub mod nft; pub mod prefunded_specialized_balance; +/// Reduced platform state stored in replicated state for state sync reconstruction +pub mod reduced_platform_state; pub mod serialization; #[cfg(any( feature = "message-signing", diff --git a/packages/rs-dpp/src/reduced_platform_state/mod.rs b/packages/rs-dpp/src/reduced_platform_state/mod.rs new file mode 100644 index 00000000000..ea17ecfc71c --- /dev/null +++ b/packages/rs-dpp/src/reduced_platform_state/mod.rs @@ -0,0 +1,106 @@ +//! Reduced platform state +//! +//! A minimal subset of the Platform state that is stored inside the replicated GroveDB +//! state (under the Misc tree), allowing a node that syncs via ABCI state sync to +//! reconstruct the full Platform state. The full Platform state itself is only persisted +//! to GroveDB aux storage, which is not replicated by GroveDB state sync. + +use crate::ProtocolError; +use bincode::{Decode, Encode}; +use derive_more::From; +use platform_serialization_derive::{PlatformDeserialize, PlatformSerialize}; + +pub mod v0; + +use v0::ReducedPlatformStateV0; + +/// Reduced Platform State (platform-versioned wrapper) +/// +/// The structure version is the enum discriminant, so it serializes `unversioned` (big +/// endian, no limit) exactly like the other versioned platform types. These bytes are +/// covered by the app hash, so the encoding is consensus-fixed. +#[derive(Clone, Debug, PartialEq, Encode, Decode, PlatformSerialize, PlatformDeserialize, From)] +#[platform_serialize(unversioned)] +pub enum ReducedPlatformState { + /// Version 0 + V0(ReducedPlatformStateV0), +} + +#[cfg(test)] +mod tests { + use super::v0::{ + ReducedBlockInfoV0, ReducedPlatformStateV0, ReducedPreviousQuorumsV0, + ReducedVerificationQuorumV0, + }; + use super::*; + use crate::block::block_info::BlockInfo; + use crate::serialization::{PlatformDeserializable, PlatformSerializable}; + + #[test] + fn should_roundtrip_reduced_platform_state_serialization() { + let state = ReducedPlatformState::V0(ReducedPlatformStateV0 { + last_committed_block_info: Some(ReducedBlockInfoV0 { + basic_info: BlockInfo::default_with_time(1_700_000_000_000), + quorum_hash: [1u8; 32].into(), + proposer_pro_tx_hash: [2u8; 32].into(), + }), + current_protocol_version_in_consensus: 15, + next_epoch_protocol_version: 15, + current_validator_set_quorum_hash: [4u8; 32].into(), + next_validator_set_quorum_hash: Some([5u8; 32].into()), + previous_fee_versions: [(0u16, 1u32)].into_iter().collect(), + quorum_positions: vec![[4u8; 32].into(), [5u8; 32].into()], + proposed_core_chain_locked_height: 1000, + previous_chain_lock_quorums: Some(ReducedPreviousQuorumsV0 { + quorums: vec![ReducedVerificationQuorumV0 { + quorum_hash: [6u8; 32].into(), + public_key: [7u8; 48], + index: None, + }], + last_active_core_height: 990, + updated_at_core_height: 995, + previous_change_height: Some(900), + }), + previous_instant_lock_quorums: Some(ReducedPreviousQuorumsV0 { + quorums: vec![ReducedVerificationQuorumV0 { + quorum_hash: [8u8; 32].into(), + public_key: [9u8; 48], + index: Some(3), + }], + last_active_core_height: 991, + updated_at_core_height: 996, + previous_change_height: None, + }), + }); + + let bytes = state.serialize_to_bytes().expect("should serialize"); + let restored = + ReducedPlatformState::deserialize_from_bytes(&bytes).expect("should deserialize"); + + assert_eq!(state, restored); + } + + /// The reduced state is encoded like every other versioned platform type: big + /// endian. Pin it so the app-hash-covered encoding cannot drift silently. + #[test] + fn should_encode_big_endian_like_other_platform_types() { + let state = ReducedPlatformState::V0(ReducedPlatformStateV0 { + last_committed_block_info: None, + current_protocol_version_in_consensus: 0x0102_0304, + next_epoch_protocol_version: 0, + current_validator_set_quorum_hash: [0u8; 32].into(), + next_validator_set_quorum_hash: None, + previous_fee_versions: Default::default(), + quorum_positions: vec![], + proposed_core_chain_locked_height: 0, + previous_chain_lock_quorums: None, + previous_instant_lock_quorums: None, + }); + + let bytes = state.serialize_to_bytes().expect("should serialize"); + // discriminant 0, then `None`, then the protocol version as a big-endian varint + // (bincode's varint marker 0xfc precedes a u32 payload) + assert_eq!(&bytes[..2], &[0u8, 0u8]); + assert_eq!(&bytes[2..7], &[0xfc, 0x01, 0x02, 0x03, 0x04]); + } +} diff --git a/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs new file mode 100644 index 00000000000..8017b1cab31 --- /dev/null +++ b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs @@ -0,0 +1,96 @@ +use crate::block::block_info::BlockInfo; +use crate::fee::default_costs::EpochIndexFeeVersionsForStorage; +use crate::util::deserializer::ProtocolVersion; +use bincode::{Decode, Encode}; +use platform_value::Bytes32; + +/// Block information persisted as part of the reduced platform state. +/// +/// Only what the block header fixes goes in here. The reduced state is written during +/// block execution and covered by the app hash, so anything that can differ between two +/// proposals of the same block (the consensus round, the app hash itself, the block id +/// hash and the signature) must stay out: a re-proposal of the same header at a later +/// round has to produce the same app hash. A state-synced node takes the app hash from +/// the snapshot and learns the rest with the next finalized block. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedBlockInfoV0 { + /// Basic block info (height, core height, time, epoch) + pub basic_info: BlockInfo, + /// The quorum that signed (or will sign) this block + pub quorum_hash: Bytes32, + /// The block proposer's pro tx hash + pub proposer_pro_tx_hash: Bytes32, +} + +/// One quorum of a signature-verification quorum set, as persisted in the reduced +/// platform state. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedVerificationQuorumV0 { + /// The quorum hash + pub quorum_hash: Bytes32, + /// The quorum's threshold BLS public key, compressed (48 bytes) + pub public_key: [u8; 48], + /// The DIP24 rotation index, for rotating quorum types + pub index: Option, +} + +/// The superseded quorums of a signature-verification quorum set, together with the core +/// heights that define the window they are still authoritative for. +/// +/// This history CANNOT be recovered from Core: `get_quorum_listextended` answers "which +/// quorums exist at height h", not "when did this node observe the set change". It is +/// nonetheless consensus-relevant — `select_quorums` picks the previous set for locks +/// signed within `SIGN_OFFSET` core blocks of a change — so it has to travel with the +/// snapshot. Without it a restored node would judge an instant lock against a different +/// quorum than a node that replayed the chain, and reject a state transition the network +/// accepted. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedPreviousQuorumsV0 { + /// The superseded quorums + pub quorums: Vec, + /// The core height at which these quorums were last active + pub last_active_core_height: u32, + /// The core height at which the quorums were changed + pub updated_at_core_height: u32, + /// The core height at which the set before these became active + pub previous_change_height: Option, +} + +/// Reduced Platform State V0. +/// +/// This minimal version of the Platform state is written into GroveDB (under the Misc +/// tree, hence below the root hash) on every block proposal. Because it is part of the +/// replicated state, a freshly state-synced node can read it back and reconstruct the +/// full in-memory Platform state, which is otherwise only persisted to non-replicated +/// GroveDB aux storage. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedPlatformStateV0 { + /// Info about the block that was being processed when this state was written + /// (it becomes the last committed block once the block finalizes) + pub last_committed_block_info: Option, + /// Current protocol version in consensus + pub current_protocol_version_in_consensus: ProtocolVersion, + /// Upcoming protocol version + pub next_epoch_protocol_version: ProtocolVersion, + /// Current validator set quorum hash + pub current_validator_set_quorum_hash: Bytes32, + /// Next validator set quorum hash + pub next_validator_set_quorum_hash: Option, + /// Fee versions of previous epochs, stored by fee version number so they can be + /// restored faithfully on reconstruction + pub previous_fee_versions: EpochIndexFeeVersionsForStorage, + /// Ordered list of quorum hashes reflecting validator set quorum positions + // TODO: optimize this to not store the whole quorum hash, but only some index + pub quorum_positions: Vec, + /// Core chain locked height, as provided in RequestProcessProposal ABCI message; + /// note this can differ from the one in RequestPrepareProposal, as it can be + /// modified by the proposer. + pub proposed_core_chain_locked_height: u32, + /// The superseded chain lock validating quorums, if any. The CURRENT set is + /// re-derived from Core during reconstruction (it is exactly the quorum list at + /// `proposed_core_chain_locked_height`); only the history has to be carried. + pub previous_chain_lock_quorums: Option, + /// The superseded instant lock validating quorums, if any. See + /// [`ReducedPreviousQuorumsV0`] for why this cannot be left to reconstruction. + pub previous_instant_lock_quorums: Option, +} diff --git a/packages/rs-drive-abci/.env.local b/packages/rs-drive-abci/.env.local index c0e3ac3347a..4eb320b87aa 100644 --- a/packages/rs-drive-abci/.env.local +++ b/packages/rs-drive-abci/.env.local @@ -90,3 +90,10 @@ GROVEDB_VISUALIZER_ENABLED=false GROVEDB_VISUALIZER_ADDRESS=127.0.0.1:8083 NETWORK=regtest + +# ABCI state sync snapshots (serving side; disabled by default) +SNAPSHOTS_ENABLED=false +SNAPSHOTS_FREQUENCY_SECONDS=600 +MAX_NUM_SNAPSHOTS=3 +# CHECKPOINTS_PATH defaults to /checkpoints when unset +#CHECKPOINTS_PATH= diff --git a/packages/rs-drive-abci/.env.mainnet b/packages/rs-drive-abci/.env.mainnet index 65409c1d0a3..214b2b6c001 100644 --- a/packages/rs-drive-abci/.env.mainnet +++ b/packages/rs-drive-abci/.env.mainnet @@ -92,3 +92,10 @@ GROVEDB_VISUALIZER_ADDRESS=127.0.0.1:8083 PROPOSER_TX_PROCESSING_TIME_LIMIT=5000 NETWORK=mainnet + +# ABCI state sync snapshots (serving side; disabled by default) +SNAPSHOTS_ENABLED=false +SNAPSHOTS_FREQUENCY_SECONDS=600 +MAX_NUM_SNAPSHOTS=3 +# CHECKPOINTS_PATH defaults to /checkpoints when unset +#CHECKPOINTS_PATH= diff --git a/packages/rs-drive-abci/.env.testnet b/packages/rs-drive-abci/.env.testnet index 9e85d109c5f..b5a8379edc6 100644 --- a/packages/rs-drive-abci/.env.testnet +++ b/packages/rs-drive-abci/.env.testnet @@ -92,3 +92,10 @@ GROVEDB_VISUALIZER_ADDRESS=127.0.0.1:8083 PROPOSER_TX_PROCESSING_TIME_LIMIT=5000 NETWORK=testnet + +# ABCI state sync snapshots (serving side; disabled by default) +SNAPSHOTS_ENABLED=false +SNAPSHOTS_FREQUENCY_SECONDS=600 +MAX_NUM_SNAPSHOTS=3 +# CHECKPOINTS_PATH defaults to /checkpoints when unset +#CHECKPOINTS_PATH= diff --git a/packages/rs-drive-abci/src/abci/app/check_tx.rs b/packages/rs-drive-abci/src/abci/app/check_tx.rs index 170eb519599..d4193d97464 100644 --- a/packages/rs-drive-abci/src/abci/app/check_tx.rs +++ b/packages/rs-drive-abci/src/abci/app/check_tx.rs @@ -2,6 +2,7 @@ use crate::abci::app::PlatformApplication; use crate::abci::handler; use crate::error::Error; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::SnapshotManager; use crate::rpc::core::CoreRPCLike; use crate::utils::spawn_blocking_task_with_name_if_supported; use async_trait::async_trait; @@ -22,6 +23,12 @@ where /// Platform platform: Arc>, core_rpc: Arc, + /// The snapshot manager, pinning checkpoints that are being served to peers. + /// + /// Shared (`Arc`) rather than owned: this application never sees blocks, so it + /// cannot expire the pins of abandoned transfers itself — `server::start` keeps a + /// clone and sweeps expired pins on a timer. + snapshot_manager: Arc, } impl PlatformApplication for CheckTxAbciApplication @@ -38,8 +45,16 @@ where C: CoreRPCLike + Send + Sync + 'static, { /// Create new ABCI app - pub fn new(platform: Arc>, core_rpc: Arc) -> Self { - Self { platform, core_rpc } + pub fn new( + platform: Arc>, + core_rpc: Arc, + snapshot_manager: Arc, + ) -> Self { + Self { + platform, + core_rpc, + snapshot_manager, + } } } @@ -92,6 +107,47 @@ where .await .map_err(|error| tonic::Status::internal(format!("check tx panics: {}", error)))? } + + async fn list_snapshots( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + // Checkpoint metadata reads are synchronous rocksdb work; requests are + // peer-controlled, so keep them off the async workers (same pattern as check_tx) + let platform = Arc::clone(&self.platform); + let proto_request = request.into_inner(); + + spawn_blocking_task_with_name_if_supported("list_snapshots", move || { + handler::list_snapshots(platform.as_ref(), proto_request) + .map(tonic::Response::new) + .map_err(error_into_status) + })? + .await + .map_err(|error| tonic::Status::internal(format!("list snapshots panics: {}", error)))? + } + + async fn load_snapshot_chunk( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + // Chunk generation traverses the checkpoint's grovedb and encodes a replication + // chunk — synchronous, potentially large, and peer-controlled. Run it on the + // blocking pool so concurrent snapshot consumers cannot occupy the async workers + // and delay unrelated gRPC traffic (same pattern as check_tx). + let platform = Arc::clone(&self.platform); + let snapshot_manager = Arc::clone(&self.snapshot_manager); + let proto_request = request.into_inner(); + + spawn_blocking_task_with_name_if_supported("load_snapshot_chunk", move || { + handler::load_snapshot_chunk(platform.as_ref(), &snapshot_manager, proto_request) + .map(tonic::Response::new) + .map_err(error_into_status) + })? + .await + .map_err(|error| { + tonic::Status::internal(format!("load snapshot chunk panics: {}", error)) + })? + } } pub fn error_into_status(error: Error) -> tonic::Status { @@ -146,7 +202,11 @@ mod tests { let core_rpc = MockCoreRPCLike::new(); - let app = CheckTxAbciApplication::new(Arc::new(platform.platform), Arc::new(core_rpc)); + let app = CheckTxAbciApplication::new( + Arc::new(platform.platform), + Arc::new(core_rpc), + Arc::new(SnapshotManager::new()), + ); let debug_str = format!("{:?}", app); assert_eq!(debug_str, ""); @@ -159,7 +219,11 @@ mod tests { let core_rpc = MockCoreRPCLike::new(); - let app = CheckTxAbciApplication::new(Arc::new(platform.platform), Arc::new(core_rpc)); + let app = CheckTxAbciApplication::new( + Arc::new(platform.platform), + Arc::new(core_rpc), + Arc::new(SnapshotManager::new()), + ); // Just verify we can call platform() without panicking let _platform_ref = app.platform(); diff --git a/packages/rs-drive-abci/src/abci/app/consensus.rs b/packages/rs-drive-abci/src/abci/app/consensus.rs index 43b6d518db8..8147a1a4f49 100644 --- a/packages/rs-drive-abci/src/abci/app/consensus.rs +++ b/packages/rs-drive-abci/src/abci/app/consensus.rs @@ -1,10 +1,13 @@ -use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; +use crate::abci::app::{ + BlockExecutionApplication, PlatformApplication, StateSyncApplication, TransactionalApplication, +}; use crate::abci::handler; use crate::abci::handler::error::error_into_exception; use crate::error::execution::ExecutionError; use crate::error::Error; use crate::execution::types::block_execution_context::BlockExecutionContext; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::SnapshotFetchingSession; use crate::rpc::core::CoreRPCLike; use dpp::version::PlatformVersion; use drive::grovedb::Transaction; @@ -23,6 +26,8 @@ pub struct ConsensusAbciApplication<'a, C> { transaction: RwLock>>, /// The current block execution context block_execution_context: RwLock>, + /// The state sync transfer currently in progress, if any + snapshot_fetching_session: RwLock>>, } impl<'a, C> ConsensusAbciApplication<'a, C> { @@ -32,6 +37,7 @@ impl<'a, C> ConsensusAbciApplication<'a, C> { platform, transaction: Default::default(), block_execution_context: Default::default(), + snapshot_fetching_session: Default::default(), } } } @@ -42,6 +48,16 @@ impl PlatformApplication for ConsensusAbciApplication<'_, C> { } } +impl<'a, C> StateSyncApplication<'a, C> for ConsensusAbciApplication<'a, C> { + fn snapshot_fetching_session(&self) -> &RwLock>> { + &self.snapshot_fetching_session + } + + fn platform(&self) -> &'a Platform { + self.platform + } +} + impl BlockExecutionApplication for ConsensusAbciApplication<'_, C> { fn block_execution_context(&self) -> &RwLock> { &self.block_execution_context @@ -105,7 +121,7 @@ mod tests { crate::test::helpers::setup::TestPlatformBuilder::new().build_with_mock_rpc(); let app = ConsensusAbciApplication::::new(&platform.platform); - let _platform_ref = app.platform(); + let _platform_ref = PlatformApplication::platform(&app); } #[test] @@ -222,4 +238,18 @@ where ) -> Result { handler::verify_vote_extension(self, request).map_err(error_into_exception) } + + fn offer_snapshot( + &self, + request: proto::RequestOfferSnapshot, + ) -> Result { + handler::offer_snapshot(self, request).map_err(error_into_exception) + } + + fn apply_snapshot_chunk( + &self, + request: proto::RequestApplySnapshotChunk, + ) -> Result { + handler::apply_snapshot_chunk(self, request).map_err(error_into_exception) + } } diff --git a/packages/rs-drive-abci/src/abci/app/full.rs b/packages/rs-drive-abci/src/abci/app/full.rs index bd290b87156..01d225a9751 100644 --- a/packages/rs-drive-abci/src/abci/app/full.rs +++ b/packages/rs-drive-abci/src/abci/app/full.rs @@ -1,10 +1,13 @@ -use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; +use crate::abci::app::{ + BlockExecutionApplication, PlatformApplication, StateSyncApplication, TransactionalApplication, +}; use crate::abci::handler; use crate::abci::handler::error::error_into_exception; use crate::error::execution::ExecutionError; use crate::error::Error; use crate::execution::types::block_execution_context::BlockExecutionContext; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::{SnapshotFetchingSession, SnapshotManager}; use crate::rpc::core::CoreRPCLike; use dpp::version::PlatformVersion; use drive::grovedb::Transaction; @@ -23,6 +26,10 @@ pub struct FullAbciApplication<'a, C> { pub transaction: RwLock>>, /// The current block execution context pub block_execution_context: RwLock>, + /// The snapshot manager, pinning checkpoints that are being served to peers + pub snapshot_manager: SnapshotManager, + /// The state sync transfer currently in progress, if any + pub snapshot_fetching_session: RwLock>>, } impl<'a, C> FullAbciApplication<'a, C> { @@ -32,6 +39,8 @@ impl<'a, C> FullAbciApplication<'a, C> { platform, transaction: Default::default(), block_execution_context: Default::default(), + snapshot_manager: SnapshotManager::new(), + snapshot_fetching_session: Default::default(), } } } @@ -42,6 +51,16 @@ impl PlatformApplication for FullAbciApplication<'_, C> { } } +impl<'a, C> StateSyncApplication<'a, C> for FullAbciApplication<'a, C> { + fn snapshot_fetching_session(&self) -> &RwLock>> { + &self.snapshot_fetching_session + } + + fn platform(&self) -> &'a Platform { + self.platform + } +} + impl BlockExecutionApplication for FullAbciApplication<'_, C> { fn block_execution_context(&self) -> &RwLock> { &self.block_execution_context @@ -105,7 +124,7 @@ mod tests { crate::test::helpers::setup::TestPlatformBuilder::new().build_with_mock_rpc(); let app = FullAbciApplication::::new(&platform.platform); - let _platform_ref = app.platform(); + let _platform_ref = PlatformApplication::platform(&app); } #[test] @@ -218,6 +237,13 @@ where &self, request: proto::RequestFinalizeBlock, ) -> Result { + // Autonomous expiry of serving pins: an abandoned state sync transfer stops + // making chunk requests, so nothing on the serving path would ever release its + // pin and the pruned checkpoint directory would stay on disk forever. A block is + // the one thing that reliably keeps happening. This is node-local bookkeeping and + // touches no consensus state. + self.snapshot_manager.release_expired_pins(); + handler::finalize_block(self, request).map_err(error_into_exception) } @@ -241,4 +267,33 @@ where ) -> Result { handler::verify_vote_extension(self, request).map_err(error_into_exception) } + + fn list_snapshots( + &self, + request: proto::RequestListSnapshots, + ) -> Result { + handler::list_snapshots(self.platform, request).map_err(error_into_exception) + } + + fn load_snapshot_chunk( + &self, + request: proto::RequestLoadSnapshotChunk, + ) -> Result { + handler::load_snapshot_chunk(self.platform, &self.snapshot_manager, request) + .map_err(error_into_exception) + } + + fn offer_snapshot( + &self, + request: proto::RequestOfferSnapshot, + ) -> Result { + handler::offer_snapshot(self, request).map_err(error_into_exception) + } + + fn apply_snapshot_chunk( + &self, + request: proto::RequestApplySnapshotChunk, + ) -> Result { + handler::apply_snapshot_chunk(self, request).map_err(error_into_exception) + } } diff --git a/packages/rs-drive-abci/src/abci/app/mod.rs b/packages/rs-drive-abci/src/abci/app/mod.rs index 27d7ef0794e..209608df0d8 100644 --- a/packages/rs-drive-abci/src/abci/app/mod.rs +++ b/packages/rs-drive-abci/src/abci/app/mod.rs @@ -10,6 +10,7 @@ pub mod execution_result; mod full; use crate::execution::types::block_execution_context::BlockExecutionContext; +use crate::platform_types::snapshot::SnapshotFetchingSession; use crate::rpc::core::DefaultCoreRPC; #[cfg(test)] pub(crate) use check_tx::error_into_status; @@ -24,6 +25,16 @@ pub trait PlatformApplication { fn platform(&self) -> &Platform; } +/// ABCI application that can bootstrap its state via state sync +pub trait StateSyncApplication<'p, C = DefaultCoreRPC> { + /// Returns the state sync transfer currently in progress, if any + fn snapshot_fetching_session(&self) -> &RwLock>>; + + /// Returns Platform with the full `'p` lifetime, so a grovedb state sync session + /// borrowing the grove can be stored in the snapshot fetching session + fn platform(&self) -> &'p Platform; +} + /// Transactional ABCI application pub trait TransactionalApplication<'a> { /// Creates and keeps a new transaction diff --git a/packages/rs-drive-abci/src/abci/config.rs b/packages/rs-drive-abci/src/abci/config.rs index 7f80ab0e010..da8bda064ab 100644 --- a/packages/rs-drive-abci/src/abci/config.rs +++ b/packages/rs-drive-abci/src/abci/config.rs @@ -1,7 +1,8 @@ //! Configuration of ABCI Application server -use crate::utils::from_opt_str_or_number; +use crate::utils::{from_opt_str_or_number, from_str_or_native}; use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; // We allow changes in the ABCI configuration, but there should be a social process // involved in making this change. @@ -37,6 +38,78 @@ pub struct AbciConfig { /// Maximum time limit (in ms) to process state transitions to prepare proposal #[serde(default, deserialize_with = "from_opt_str_or_number")] pub proposer_tx_processing_time_limit: Option, + + /// State sync snapshot serving configuration + #[serde(flatten)] + pub state_sync: StateSyncAbciConfig, +} + +/// Configuration of ABCI state sync snapshot serving. +/// +/// NOTE: the field names (and thus the environment variable names `SNAPSHOTS_ENABLED`, +/// `SNAPSHOTS_FREQUENCY_SECONDS`, `MAX_NUM_SNAPSHOTS`, `CHECKPOINTS_PATH`) are a contract +/// with dashmate's env generation — do not rename them. +// @append_only +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StateSyncAbciConfig { + /// Whether snapshots are offered to state-syncing peers. When enabled, the + /// snapshot frequency and retention below override the platform-version-driven + /// checkpoint parameters. + #[serde( + default = "StateSyncAbciConfig::default_snapshots_enabled", + deserialize_with = "from_str_or_native" + )] + pub snapshots_enabled: bool, + + /// How often (in seconds) a snapshot (grovedb checkpoint) is created + #[serde( + default = "StateSyncAbciConfig::default_snapshots_frequency_seconds", + deserialize_with = "from_str_or_native" + )] + pub snapshots_frequency_seconds: u32, + + /// Maximum number of snapshots kept on disk + #[serde( + default = "StateSyncAbciConfig::default_max_num_snapshots", + deserialize_with = "from_str_or_native" + )] + pub max_num_snapshots: usize, + + /// Directory where checkpoints are stored; defaults to `/checkpoints` + #[serde(default)] + pub checkpoints_path: Option, +} + +impl StateSyncAbciConfig { + pub(crate) fn default_snapshots_enabled() -> bool { + false + } + + pub(crate) fn default_snapshots_frequency_seconds() -> u32 { + 600 + } + + pub(crate) fn default_max_num_snapshots() -> usize { + 3 + } + + /// Resolves the checkpoints directory, defaulting to `/checkpoints` + pub fn resolved_checkpoints_path(&self, db_path: &Path) -> PathBuf { + self.checkpoints_path + .clone() + .unwrap_or_else(|| db_path.join("checkpoints")) + } +} + +impl Default for StateSyncAbciConfig { + fn default() -> Self { + Self { + snapshots_enabled: Self::default_snapshots_enabled(), + snapshots_frequency_seconds: Self::default_snapshots_frequency_seconds(), + max_num_snapshots: Self::default_max_num_snapshots(), + checkpoints_path: None, + } + } } impl AbciConfig { @@ -58,6 +131,7 @@ impl Default for AbciConfig { chain_id: "chain_id".to_string(), log: Default::default(), proposer_tx_processing_time_limit: Default::default(), + state_sync: Default::default(), } } } @@ -85,6 +159,42 @@ mod tests { assert_eq!(config.chain_id, "chain_id"); assert!(config.log.is_empty()); assert!(config.proposer_tx_processing_time_limit.is_none()); + assert!(!config.state_sync.snapshots_enabled); + assert_eq!(config.state_sync.snapshots_frequency_seconds, 600); + assert_eq!(config.state_sync.max_num_snapshots, 3); + assert!(config.state_sync.checkpoints_path.is_none()); + } + + #[test] + fn state_sync_config_resolves_default_checkpoints_path_from_db_path() { + let config = StateSyncAbciConfig::default(); + assert_eq!( + config.resolved_checkpoints_path(Path::new("/var/lib/drive/db")), + PathBuf::from("/var/lib/drive/db/checkpoints") + ); + + let config = StateSyncAbciConfig { + checkpoints_path: Some(PathBuf::from("/mnt/checkpoints")), + ..Default::default() + }; + assert_eq!( + config.resolved_checkpoints_path(Path::new("/var/lib/drive/db")), + PathBuf::from("/mnt/checkpoints") + ); + } + + #[test] + fn state_sync_config_deserializes_from_env_style_strings() { + // envy provides every value as a string; the custom deserializers must coerce + let json = r#"{"abci_consensus_bind_address": "tcp://x:1", "snapshots_enabled": "true", "snapshots_frequency_seconds": "120", "max_num_snapshots": "5", "checkpoints_path": "/tmp/checkpoints"}"#; + let config: AbciConfig = serde_json::from_str(json).expect("should deserialize"); + assert!(config.state_sync.snapshots_enabled); + assert_eq!(config.state_sync.snapshots_frequency_seconds, 120); + assert_eq!(config.state_sync.max_num_snapshots, 5); + assert_eq!( + config.state_sync.checkpoints_path, + Some(PathBuf::from("/tmp/checkpoints")) + ); } #[test] @@ -98,6 +208,7 @@ mod tests { chain_id: "test-chain".to_string(), log: Default::default(), proposer_tx_processing_time_limit: None, + state_sync: Default::default(), }; let serialized = serde_json::to_string(&config).expect("should serialize"); @@ -143,6 +254,7 @@ mod tests { chain_id: "clone-test".to_string(), log: Default::default(), proposer_tx_processing_time_limit: Some(1000), + state_sync: Default::default(), }; let cloned = config.clone(); diff --git a/packages/rs-drive-abci/src/abci/error.rs b/packages/rs-drive-abci/src/abci/error.rs index 306e0644956..ae1678218c3 100644 --- a/packages/rs-drive-abci/src/abci/error.rs +++ b/packages/rs-drive-abci/src/abci/error.rs @@ -54,6 +54,14 @@ pub enum AbciError { #[error("bad commit signature: {0}")] BadCommitSignature(String), + /// Invalid state sync request received from Tenderdash or a peer + #[error("bad request state sync: {0}")] + StateSyncBadRequest(String), + + /// Internal error during state sync + #[error("internal error state sync: {0}")] + StateSyncInternalError(String), + /// The chain lock received was invalid #[error("invalid chain lock: {0}")] InvalidChainLock(String), @@ -90,6 +98,12 @@ pub enum AbciError { /// Generic with code should only be used in tests #[error("invalid state transition error: {0}")] InvalidStateTransition(#[from] ConsensusError), + + /// The current state was restored via state sync and carries no block id hash or + /// quorum signature yet, so a proof built from it could never authenticate; the + /// metadata arrives with the first block finalized after the restore. + #[error("state sync proof metadata not yet available: {0}")] + StateSyncProofMetadataUnavailable(String), } #[cfg(test)] diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs new file mode 100644 index 00000000000..74120b1db2d --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -0,0 +1,496 @@ +use crate::abci::app::StateSyncApplication; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::snapshot::{ + clear_restore_sentinel_best_effort, wipe_drive_for_restore, MAX_STATE_SYNC_CHUNK_ID_SIZE, + MAX_STATE_SYNC_CHUNK_SIZE, +}; +use crate::rpc::core::CoreRPCLike; +use std::sync::atomic::Ordering; +use tenderdash_abci::proto::abci as proto; +use tenderdash_abci::proto::abci::response_apply_snapshot_chunk; + +/// Applies one chunk of a state sync snapshot to the grovedb sync session. +/// +/// A chunk grovedb rejects does not kill the whole transfer: the sender is banned and +/// Tenderdash is asked to restart the snapshot (grovedb invalidates the session on a +/// failed chunk, so the restore starts over from a fresh session on the re-offer). When +/// the last chunk lands, the session is committed, grovedb is verified against the +/// target app hash, and the platform state is reconstructed from the reduced platform +/// state contained in the restored snapshot. +pub fn apply_snapshot_chunk<'a, 'db: 'a, A, C>( + app: &'a A, + request: proto::RequestApplySnapshotChunk, +) -> Result +where + A: StateSyncApplication<'db, C> + 'db, + C: CoreRPCLike + 'db, +{ + tracing::trace!( + chunk_id = hex::encode(&request.chunk_id), + chunk_len = request.chunk.len(), + "[state_sync] api apply_snapshot_chunk", + ); + + let mut session_write_guard = app.snapshot_fetching_session().write().map_err(|_| { + AbciError::StateSyncInternalError( + "apply_snapshot_chunk unable to lock session (poisoned)".to_string(), + ) + })?; + + // The version the SNAPSHOT was produced at, pinned when the offer was accepted. Using + // `self.state`'s version here would be wrong: a state-syncing node has no saved state, + // so it is still on the initial protocol version and would decode, verify and hash the + // restored trees under a different (older) grovedb table than the one that generated + // the chunks. + let platform_version = session_write_guard + .as_ref() + .ok_or(AbciError::StateSyncBadRequest( + "apply_snapshot_chunk no state sync session in progress".to_string(), + ))? + .platform_version; + let grove_version = &platform_version.drive.grove_version; + + { + let session = session_write_guard + .as_mut() + .expect("session presence was just checked"); + + let reject_senders = if request.sender.is_empty() { + vec![] + } else { + vec![request.sender.clone()] + }; + + // Cap peer-supplied sizes before anything decodes them (issue #3773). + // + // These are TRANSFER faults, not reasons to abort state sync: an application + // error here would reach Tenderdash as an ABCI exception, killing the whole + // restore and leaving the node on the wiped database the offer created (with the + // restore sentinel still set). Both caps are therefore answered with a + // recoverable response; they run before grovedb sees the chunk, so unlike a + // chunk grovedb rejects (below) they leave the session usable. + if request.chunk.len() > MAX_STATE_SYNC_CHUNK_SIZE { + // Oversized chunk DATA: the chunk id itself is still fine, so ban the sender + // and have Tenderdash refetch exactly this chunk from someone else. + tracing::warn!( + chunk_id = hex::encode(&request.chunk_id), + sender = request.sender, + chunk_len = request.chunk.len(), + limit = MAX_STATE_SYNC_CHUNK_SIZE, + "[state_sync] apply_snapshot_chunk oversized chunk, rejecting the sender and requesting refetch", + ); + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::Retry.into(), + refetch_chunks: vec![request.chunk_id], + reject_senders, + next_chunks: vec![], + }); + } + if request.chunk_id.len() > MAX_STATE_SYNC_CHUNK_ID_SIZE { + // An oversized chunk id is intrinsically invalid — refetching it would ask for + // the same impossible id again — so restart the snapshot instead. + tracing::warn!( + chunk_id_len = request.chunk_id.len(), + sender = request.sender, + limit = MAX_STATE_SYNC_CHUNK_ID_SIZE, + "[state_sync] apply_snapshot_chunk oversized chunk id, requesting snapshot restart", + ); + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::RetrySnapshot.into(), + refetch_chunks: vec![], + reject_senders, + next_chunks: vec![], + }); + } + + let wire_version = session.wire_version; + let next_chunk_ids = match session.state_sync_info.apply_chunk( + &request.chunk_id, + &request.chunk, + wire_version, + grove_version, + ) { + Ok(next_chunk_ids) => next_chunk_ids, + Err(e) => { + // A chunk grovedb cannot apply (corrupted or tampered data) permanently + // invalidates the grovedb session: every later `apply_chunk` and the + // final commit refuse, and grovedb does not expose whether a given error + // poisoned the session or was caught before any write. The transfer is + // still recoverable — ban the sender and ask Tenderdash to restart the + // snapshot (a re-offer, which `offer_snapshot` answers by wiping and + // opening a fresh session) rather than refetch a chunk this session can + // no longer accept. + tracing::warn!( + chunk_id = hex::encode(&request.chunk_id), + sender = request.sender, + error = ?e, + "[state_sync] apply_snapshot_chunk rejected a chunk, requesting snapshot restart", + ); + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::RetrySnapshot.into(), + refetch_chunks: vec![], + reject_senders, + next_chunks: vec![], + }); + } + }; + + if !session.state_sync_info.is_sync_completed() { + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::Accept.into(), + refetch_chunks: vec![], + reject_senders: vec![], + next_chunks: next_chunk_ids, + }); + } + + if !next_chunk_ids.is_empty() { + return Err(AbciError::StateSyncInternalError( + "apply_snapshot_chunk session is completed but next_chunk_ids is not empty" + .to_string(), + ) + .into()); + } + } + + // The transfer is complete: consume the session and commit it + let session = session_write_guard + .take() + .expect("session presence was just checked"); + + // grovedb only makes the session durable once its own root-hash check passes, so a + // failure here leaves nothing committed — but the database is still WIPED from the + // offer, so the node cannot be left as it is. Route it through the same recovery path + // as every later failure, which also keeps Tenderdash's snapshot ladder moving instead + // of aborting state sync with an exception. + if let Err(e) = app + .platform() + .drive + .grove + .commit_session(session.state_sync_info, grove_version) + { + return reject_restored_snapshot(app, &format!("unable to commit the session: {}", e)); + } + + tracing::debug!("[state_sync] transfer complete, verifying grovedb"); + + // From here on the session is COMMITTED: grovedb durably holds the restored state + // while the platform state still describes the node from before the sync. Every + // failure below must therefore go through `reject_restored_snapshot`, which puts the + // node back to an empty, self-consistent slate and asks Tenderdash for another + // snapshot. Returning an error instead would leave the node holding a database its + // platform state knows nothing about, and the `info` handler panics on exactly that + // mismatch — a crash loop that no restart can clear. + let incorrect_hashes = + match app + .platform() + .drive + .grove + .verify_grovedb(None, true, false, grove_version) + { + Ok(incorrect_hashes) => incorrect_hashes, + Err(e) => { + return reject_restored_snapshot( + app, + &format!("unable to verify the restored grovedb: {}", e), + ); + } + }; + if !incorrect_hashes.is_empty() { + let paths: Vec = incorrect_hashes + .keys() + .take(5) + .map(|path| path.iter().map(hex::encode).collect::>().join("/")) + .collect(); + return reject_restored_snapshot( + app, + &format!( + "grovedb verification failed with {} incorrect hashes, first paths: [{}]", + incorrect_hashes.len(), + paths.join(", ") + ), + ); + } + + // Rebuild the in-memory platform state from the reduced platform state contained in + // the restored snapshot. This re-derives masternode lists and quorums from Core in + // memory only; the root hash equality check below is the restore's integrity backstop. + // + // This is also where a snapshot taken before the reduced platform state existed + // (pre-v15) is refused. Refusing earlier would be better, but grovedb does not expose + // the session's transaction, so the Misc tree cannot be probed before the commit — + // see the note on `reject_restored_snapshot`. + if let Err(e) = app + .platform() + .reconstruct_platform_state(&session.app_hash, platform_version) + { + return reject_restored_snapshot( + app, + &format!("unable to reconstruct the platform state: {}", e), + ); + } + + let drive_app_hash = match app + .platform() + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + { + Ok(drive_app_hash) => drive_app_hash, + Err(e) => { + return reject_restored_snapshot( + app, + &format!("unable to get the restored app hash: {}", e), + ); + } + }; + + if drive_app_hash != session.app_hash { + tracing::error!( + state_sync_app_hash = hex::encode(session.app_hash), + drive_app_hash = hex::encode(drive_app_hash), + "[state_sync] restored grovedb root hash does not match the snapshot app hash", + ); + return reject_restored_snapshot( + app, + &format!( + "grovedb verification failed with incorrect app hash: {}", + hex::encode(drive_app_hash) + ), + ); + } + + // The query service only serves while `committed_block_height_guard` matches the + // published state's height. A fresh node's guard is still 0 (nothing was ever + // finalized through it), while `reconstruct_platform_state` just published the + // state at the snapshot height — left alone, that mismatch keeps every query + // unserviceable until the first post-restore block finalizes. Open the gate only + // HERE, after grovedb reconstruction, aux persistence and the final app-hash check + // have all succeeded: on any earlier failure the guard stays 0, exactly as it must + // for a node whose restore was rejected. (A restart re-derives the guard from the + // persisted state, so this store also matches what the next boot would compute.) + app.platform().committed_block_height_guard.store( + app.platform().state.load().last_committed_block_height(), + Ordering::Relaxed, + ); + + // The restore is complete and the node is self-consistent again, so the marker that + // tells a restarting process to wipe can go. This is deliberately the LAST step, after + // `reconstruct_platform_state` has committed the platform state to aux storage, and + // deliberately best-effort: a successful restore must not be turned into an ABCI error + // by a `remove_file` hiccup. + clear_restore_sentinel_best_effort(&app.platform().config.db_path); + + tracing::info!( + height = session.snapshot.height, + app_hash = hex::encode(session.app_hash), + "state_sync completed", + ); + + Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::CompleteSnapshot.into(), + refetch_chunks: vec![], + reject_senders: vec![], + next_chunks: vec![], + }) +} + +/// Puts the node back to an empty, self-consistent slate after a restore that was already +/// committed to grovedb turned out to be unusable, and asks Tenderdash to try a different +/// snapshot. +/// +/// Ideally an unusable snapshot would be detected BEFORE `commit_session`, by probing the +/// Misc tree through the session's still-open transaction. grovedb keeps that transaction +/// private (`MultiStateSyncSession::transaction`, no accessor), so there is no way to read +/// the restored state before it lands. Until grovedb exposes it, this is the containment: +/// undo the commit by wiping, and let Tenderdash pick another snapshot. +/// +/// The restore sentinel is deliberately LEFT IN PLACE. The database is empty, but the +/// in-memory platform state may still describe the chain the offer wiped, so the node is +/// not yet provably consistent. Everything that can happen next resolves it: another +/// `offer_snapshot` re-wipes and re-marks, a successful restore clears it, an `init_chain` +/// clears it, and a restart before any of those wipes and comes up empty. +/// +/// `REJECT_SNAPSHOT` rather than an error is what keeps Tenderdash walking its ladder: it +/// discards this snapshot, tries the next, and falls back to block sync when it runs out. +/// An ABCI exception here would abort state sync altogether. +fn reject_restored_snapshot<'a, 'db: 'a, A, C>( + app: &'a A, + reason: &str, +) -> Result +where + A: StateSyncApplication<'db, C> + 'db, + C: CoreRPCLike + 'db, +{ + tracing::error!( + reason, + "[state_sync] restored snapshot is unusable, wiping and asking for another one", + ); + + wipe_drive_for_restore(&app.platform().drive).map_err(|e| { + AbciError::StateSyncInternalError(format!( + "apply_snapshot_chunk unable to wipe after rejecting a snapshot ({}): {}", + reason, e + )) + })?; + + Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::RejectSnapshot.into(), + refetch_chunks: vec![], + reject_senders: vec![], + next_chunks: vec![], + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::abci::handler::offer_snapshot; + use crate::platform_types::snapshot::encode_snapshot_metadata; + use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::version::v15::PROTOCOL_VERSION_15; + use tenderdash_abci::proto::abci::response_offer_snapshot; + + #[test] + fn apply_snapshot_chunk_without_session_is_rejected() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + assert!(apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: vec![1u8; 32], + chunk: vec![], + sender: String::new(), + }, + ) + .is_err()); + } + + fn offer_a_snapshot(app: &FullAbciApplication) -> Vec { + let target_app_hash = vec![7u8; 32]; + offer_snapshot( + app, + proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 100, + version: 1, + hash: target_app_hash.clone(), + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), + }), + app_hash: target_app_hash.clone(), + }, + ) + .expect("should accept offer"); + target_app_hash + } + + /// An oversized chunk or chunk id is a recoverable transfer fault, not a reason to + /// abort state sync with an ABCI exception: the session must survive and Tenderdash + /// must be given a way forward. + #[test] + fn apply_snapshot_chunk_caps_sizes_without_aborting_the_transfer() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + let target_app_hash = offer_a_snapshot(&app); + + // Oversized chunk data: ban the sender and refetch exactly this chunk + let response = apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: target_app_hash.clone(), + chunk: vec![0u8; MAX_STATE_SYNC_CHUNK_SIZE + 1], + sender: "fat-peer".to_string(), + }, + ) + .expect("an oversized chunk must not abort the ABCI request"); + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::Retry) + ); + assert_eq!(response.refetch_chunks, vec![target_app_hash]); + assert_eq!(response.reject_senders, vec!["fat-peer".to_string()]); + assert!( + app.snapshot_fetching_session.read().unwrap().is_some(), + "the session must survive an oversized chunk" + ); + + // Oversized chunk id: intrinsically invalid, so restart the snapshot + let response = apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: vec![0u8; MAX_STATE_SYNC_CHUNK_ID_SIZE + 1], + chunk: vec![], + sender: "fat-peer".to_string(), + }, + ) + .expect("an oversized chunk id must not abort the ABCI request"); + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::RetrySnapshot) + ); + assert!(response.refetch_chunks.is_empty()); + assert_eq!(response.reject_senders, vec!["fat-peer".to_string()]); + } + + #[test] + fn apply_snapshot_chunk_asks_for_a_snapshot_restart_on_a_bad_chunk() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + let target_app_hash = offer_a_snapshot(&app); + + // Garbage bytes for the root chunk: grovedb rejects them and invalidates its + // session, so the answer is a snapshot restart that bans the sender — not an + // ABCI exception, and not a refetch this session could no longer apply. + let response = apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: target_app_hash.clone(), + chunk: vec![0xde, 0xad, 0xbe, 0xef], + sender: "peer-1".to_string(), + }, + ) + .expect("bad chunk should not error the session"); + + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::RetrySnapshot) + ); + assert!(response.refetch_chunks.is_empty()); + assert_eq!(response.reject_senders, vec!["peer-1".to_string()]); + assert!( + app.snapshot_fetching_session.read().unwrap().is_some(), + "the session stays in place until the re-offer replaces it" + ); + + // The re-offer Tenderdash answers with must be accepted and start over + let response = offer_snapshot( + &app, + proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 100, + version: 1, + hash: target_app_hash.clone(), + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), + }), + app_hash: target_app_hash, + }, + ) + .expect("re-offer after a bad chunk must not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + } +} diff --git a/packages/rs-drive-abci/src/abci/handler/init_chain.rs b/packages/rs-drive-abci/src/abci/handler/init_chain.rs index 0573b05e6b3..5923aefd11f 100644 --- a/packages/rs-drive-abci/src/abci/handler/init_chain.rs +++ b/packages/rs-drive-abci/src/abci/handler/init_chain.rs @@ -1,5 +1,6 @@ use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; use crate::error::Error; +use crate::platform_types::snapshot::clear_restore_sentinel_best_effort; use crate::rpc::core::CoreRPCLike; use tenderdash_abci::proto::abci as proto; @@ -32,6 +33,16 @@ where let app_hash = hex::encode(&response.app_hash); + // Genesis has just been created, so the node is self-consistent again. If a state sync + // restore had been abandoned (every offered snapshot rejected, Tenderdash falling back + // to block sync), its marker is still on disk and would make the NEXT restart wipe this + // perfectly good chain. Clear it here — this is the block-sync arm of the same recovery + // that `Platform::open_with_client` performs for an interrupted restore. + // Best-effort: failing to remove a marker file must not turn a working genesis into a + // failed init_chain. The worst case is one unnecessary wipe-and-resync on a later + // restart, which is loud but safe. + clear_restore_sentinel_best_effort(&app.platform().config.db_path); + tracing::info!( app_hash, chain_id, diff --git a/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs new file mode 100644 index 00000000000..2eab450dafe --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs @@ -0,0 +1,172 @@ +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::encode_snapshot_metadata; +use tenderdash_abci::proto::abci as proto; + +/// Lists the state sync snapshots this node can serve. +/// +/// Snapshots are the rocksdb checkpoints Drive already keeps (`drive.checkpoints`). +/// Only checkpoints that contain the reduced platform state are offered: a checkpoint +/// taken before the protocol version that introduced it (v15) cannot be restored, since +/// a state-synced node would have no way to reconstruct its platform state. +/// +/// Takes the platform directly rather than an application trait so the gRPC serving +/// application can run it on the blocking pool from an owned `Arc`. +pub fn list_snapshots( + platform: &Platform, + _request: proto::RequestListSnapshots, +) -> Result { + tracing::trace!("[state_sync] api list_snapshots called"); + + if !platform.config.abci.state_sync.snapshots_enabled { + return Ok(Default::default()); + } + + let checkpoints = platform.drive.checkpoints.load(); + + let mut snapshots = Vec::new(); + for (height, checkpoint_info) in checkpoints.iter() { + let checkpoint = &checkpoint_info.checkpoint; + + // Read the checkpoint under the version IT was written at, not this node's + // current one: a node that has since upgraded still serves older checkpoints, and + // grovedb's tree opening and root-hash rules are version gated. The same version + // is stamped into the snapshot metadata so the consuming node — which has no way + // to derive it — restores under exactly these rules. + let Some(snapshot_platform_version) = checkpoint.platform_version().map_err(|e| { + AbciError::StateSyncInternalError(format!( + "list_snapshots unable to read the protocol version of the checkpoint at \ + height {}: {}", + height, e + )) + })? + else { + // Unreachable on a healthy node — `store_platform_state` writes the protocol + // version in the block transaction that commits before the checkpoint is + // taken — so say so rather than silently dropping the snapshot from the list. + tracing::warn!( + height, + "[state_sync] not offering the checkpoint at this height: it records no \ + protocol version, or one this binary does not know", + ); + continue; + }; + let grove_version = &snapshot_platform_version.drive.grove_version; + + let restorable = checkpoint + .has_reduced_platform_state(grove_version) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "list_snapshots unable to inspect checkpoint at height {}: {}", + height, e + )) + })?; + if !restorable { + continue; + } + + let root_hash = checkpoint + .grove_db + .root_hash(None, grove_version) + .unwrap() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "list_snapshots unable to get root hash of checkpoint at height {}: {}", + height, e + )) + })?; + + snapshots.push(proto::Snapshot { + height: *height, + version: snapshot_platform_version + .drive_abci + .state_sync + .protocol_version as u32, + hash: root_hash.to_vec(), + metadata: encode_snapshot_metadata(snapshot_platform_version.protocol_version), + }); + } + + Ok(proto::ResponseListSnapshots { snapshots }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::PlatformConfig; + use crate::test::helpers::fast_forward_to_block::fast_forward_to_block; + use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::version::PlatformVersion; + + fn config_with_snapshots_enabled() -> PlatformConfig { + let mut config = PlatformConfig::default_local(); + config.abci.state_sync.snapshots_enabled = true; + config + } + + #[test] + fn list_snapshots_returns_nothing_when_serving_is_disabled() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let response = + list_snapshots(&platform, Default::default()).expect("should list snapshots"); + assert!(response.snapshots.is_empty()); + } + + #[test] + fn list_snapshots_serves_only_checkpoints_with_reduced_platform_state() { + let platform = TestPlatformBuilder::new() + .with_config(config_with_snapshots_enabled()) + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + + // A checkpoint taken before the reduced platform state exists (pre-v15 + // activation) is unrestorable and must not be offered. + fast_forward_to_block(&platform, 1_000_000, 10, 42, 0, false); + platform + .create_grovedb_checkpoint(platform_version) + .expect("should create checkpoint"); + + let response = + list_snapshots(&platform, Default::default()).expect("should list snapshots"); + assert!( + response.snapshots.is_empty(), + "checkpoints without the reduced platform state must be filtered out" + ); + + // Once the reduced platform state is in the replicated state, new checkpoints + // are restorable and must be offered. + let reduced_platform_state = platform.state.load().to_reduced_platform_state(None, 42); + platform + .store_reduced_platform_state(&reduced_platform_state, None, platform_version) + .expect("should store reduced platform state"); + + // A real node always has its protocol version in aux (Drive::open reads it to + // decide whether there is saved state at all); snapshots are stamped with the + // checkpoint's own version, so the test platform has to have one too. + platform + .drive + .store_current_protocol_version(platform_version.protocol_version, None) + .expect("should store protocol version"); + + fast_forward_to_block(&platform, 2_000_000, 20, 43, 0, false); + platform + .create_grovedb_checkpoint(platform_version) + .expect("should create checkpoint"); + + let response = + list_snapshots(&platform, Default::default()).expect("should list snapshots"); + assert_eq!(response.snapshots.len(), 1); + let snapshot = &response.snapshots[0]; + assert_eq!(snapshot.height, 20); + assert_eq!( + snapshot.version, + platform_version.drive_abci.state_sync.protocol_version as u32 + ); + assert_eq!(snapshot.hash.len(), 32); + } +} diff --git a/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs new file mode 100644 index 00000000000..688e70ff39f --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs @@ -0,0 +1,210 @@ +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::{ + max_serving_pins, SnapshotManager, MAX_STATE_SYNC_CHUNK_ID_SIZE, + SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, +}; +use std::sync::Arc; +use tenderdash_abci::proto::abci as proto; + +/// Serves one chunk of a state sync snapshot from the checkpoint registry. +/// +/// The served checkpoint is pinned in the snapshot manager so checkpoint pruning cannot +/// delete it from disk while a peer is still downloading it. +/// +/// Takes the platform and snapshot manager directly rather than an application trait so +/// the gRPC serving application can run it on the blocking pool from owned `Arc`s. +pub fn load_snapshot_chunk( + platform: &Platform, + snapshot_manager: &SnapshotManager, + request: proto::RequestLoadSnapshotChunk, +) -> Result { + tracing::trace!( + height = request.height, + version = request.version, + chunk_id = hex::encode(&request.chunk_id), + "[state_sync] api load_snapshot_chunk", + ); + + if !platform.config.abci.state_sync.snapshots_enabled { + return Err(AbciError::StateSyncBadRequest( + "load_snapshot_chunk snapshot serving is disabled".to_string(), + ) + .into()); + } + + // Cap peer-supplied sizes before anything decodes them (issue #3773) + if request.chunk_id.len() > MAX_STATE_SYNC_CHUNK_ID_SIZE { + return Err(AbciError::StateSyncBadRequest(format!( + "load_snapshot_chunk chunk id of {} bytes exceeds the {} byte limit", + request.chunk_id.len(), + MAX_STATE_SYNC_CHUNK_ID_SIZE + )) + .into()); + } + + let wire_version = u16::try_from(request.version) + .ok() + .filter(|version| SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS.contains(version)); + let Some(wire_version) = wire_version else { + return Err(AbciError::StateSyncBadRequest(format!( + "load_snapshot_chunk unsupported state sync protocol version {}, supported: {:?}", + request.version, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS + )) + .into()); + }; + + // Resolve the checkpoint: from the registry, or — if pruning already dropped it — + // from the pins of transfers already in flight. + let checkpoint = platform + .drive + .checkpoints + .load() + .get(&request.height) + .map(|checkpoint_info| Arc::clone(&checkpoint_info.checkpoint)) + .or_else(|| snapshot_manager.pinned_checkpoint(request.height)) + .ok_or_else(|| { + AbciError::StateSyncBadRequest(format!( + "load_snapshot_chunk no snapshot at height {}", + request.height + )) + })?; + + // Chunks must be generated under the version the checkpoint was WRITTEN at — the same + // one `list_snapshots` stamped into the snapshot metadata and the consuming node + // restores under. This node's own current version may have moved on since. + let snapshot_platform_version = checkpoint + .platform_version() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "load_snapshot_chunk unable to read the protocol version of the checkpoint at \ + height {}: {}", + request.height, e + )) + })? + .ok_or_else(|| { + AbciError::StateSyncInternalError(format!( + "load_snapshot_chunk checkpoint at height {} has no usable protocol version", + request.height + )) + })?; + let grove_version = &snapshot_platform_version.drive.grove_version; + + let chunk = checkpoint + .grove_db + .fetch_chunk(&request.chunk_id, None, wire_version, grove_version) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "load_snapshot_chunk unable to fetch chunk: {}", + e + )) + })?; + + // Pin (or refresh the pin of) the checkpoint only once a chunk was actually served. + // Pinning before the fetch would let a peer keep a checkpoint — and its directory — + // alive with a stream of requests that never succeed. + snapshot_manager.pin_for_serving( + request.height, + checkpoint, + max_serving_pins(platform.config.abci.state_sync.max_num_snapshots), + ); + + Ok(proto::ResponseLoadSnapshotChunk { chunk }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::PlatformConfig; + use crate::test::helpers::fast_forward_to_block::fast_forward_to_block; + use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::version::PlatformVersion; + + #[test] + fn load_snapshot_chunk_serves_root_chunk_and_rejects_bad_requests() { + let mut config = PlatformConfig::default_local(); + config.abci.state_sync.snapshots_enabled = true; + let platform = TestPlatformBuilder::new() + .with_config(config) + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + let snapshot_manager = SnapshotManager::new(); + + let reduced_platform_state = platform.state.load().to_reduced_platform_state(None, 42); + platform + .store_reduced_platform_state(&reduced_platform_state, None, platform_version) + .expect("should store reduced platform state"); + // Snapshots are served under the checkpoint's OWN protocol version, which a real + // node always has in aux. + platform + .drive + .store_current_protocol_version(platform_version.protocol_version, None) + .expect("should store protocol version"); + + fast_forward_to_block(&platform, 1_000_000, 10, 42, 0, false); + platform + .create_grovedb_checkpoint(platform_version) + .expect("should create checkpoint"); + + let root_hash = platform + .drive + .grove + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("should get root hash"); + + // The root chunk (chunk id == app hash) must be served + let response = load_snapshot_chunk( + &platform, + &snapshot_manager, + proto::RequestLoadSnapshotChunk { + height: 10, + version: 1, + chunk_id: root_hash.to_vec(), + }, + ) + .expect("should load root chunk"); + assert!(!response.chunk.is_empty()); + + // The served checkpoint must now be pinned against pruning + assert!(snapshot_manager.pinned_checkpoint(10).is_some()); + + // Unknown height is rejected + assert!(load_snapshot_chunk( + &platform, + &snapshot_manager, + proto::RequestLoadSnapshotChunk { + height: 999, + version: 1, + chunk_id: root_hash.to_vec(), + }, + ) + .is_err()); + + // Unsupported wire version is rejected + assert!(load_snapshot_chunk( + &platform, + &snapshot_manager, + proto::RequestLoadSnapshotChunk { + height: 10, + version: 2, + chunk_id: root_hash.to_vec(), + }, + ) + .is_err()); + + // Oversized chunk id is rejected before any decoding + assert!(load_snapshot_chunk( + &platform, + &snapshot_manager, + proto::RequestLoadSnapshotChunk { + height: 10, + version: 1, + chunk_id: vec![0u8; MAX_STATE_SYNC_CHUNK_ID_SIZE + 1], + }, + ) + .is_err()); + } +} diff --git a/packages/rs-drive-abci/src/abci/handler/mod.rs b/packages/rs-drive-abci/src/abci/handler/mod.rs index 8acd0737ebe..443758734a3 100644 --- a/packages/rs-drive-abci/src/abci/handler/mod.rs +++ b/packages/rs-drive-abci/src/abci/handler/mod.rs @@ -35,6 +35,7 @@ //! can only make changes that are backwards compatible. Otherwise new calls must be made instead. //! +mod apply_snapshot_chunk; mod check_tx; mod echo; pub mod error; @@ -42,16 +43,23 @@ mod extend_vote; mod finalize_block; mod info; mod init_chain; +mod list_snapshots; +mod load_snapshot_chunk; +mod offer_snapshot; mod prepare_proposal; mod process_proposal; mod verify_vote_extension; +pub use apply_snapshot_chunk::apply_snapshot_chunk; pub use check_tx::check_tx; pub use echo::echo; pub use extend_vote::extend_vote; pub use finalize_block::finalize_block; pub use info::info; pub use init_chain::init_chain; +pub use list_snapshots::list_snapshots; +pub use load_snapshot_chunk::load_snapshot_chunk; +pub use offer_snapshot::offer_snapshot; pub use prepare_proposal::prepare_proposal; pub use process_proposal::process_proposal; pub use verify_vote_extension::verify_vote_extension; diff --git a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs new file mode 100644 index 00000000000..d012da3ff90 --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs @@ -0,0 +1,300 @@ +use crate::abci::app::StateSyncApplication; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::snapshot::{ + decode_snapshot_metadata, wipe_drive_for_restore, write_restore_sentinel, + SnapshotFetchingSession, STATE_SYNC_SUBTREES_BATCH_SIZE, + SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, +}; +use crate::rpc::core::CoreRPCLike; +use dpp::version::v15::PROTOCOL_VERSION_15; +use dpp::version::PlatformVersion; +use tenderdash_abci::proto::abci as proto; +use tenderdash_abci::proto::abci::response_offer_snapshot; + +/// Handles a snapshot offered by Tenderdash during state sync. +/// +/// Accepting an offer wipes the local grovedb and opens a grovedb state sync session +/// targeting the light-client-verified app hash. Any accepted-format offer replaces a +/// session already in progress (also answered with Accept), whatever height it carries. +pub fn offer_snapshot<'a, 'db: 'a, A, C>( + app: &'a A, + request: proto::RequestOfferSnapshot, +) -> Result +where + A: StateSyncApplication<'db, C> + 'db, + C: CoreRPCLike + 'db, +{ + let request_app_hash: [u8; 32] = request.app_hash.try_into().map_err(|_| { + AbciError::StateSyncBadRequest("offer_snapshot invalid app_hash length".to_string()) + })?; + let offered_snapshot = request.snapshot.ok_or(AbciError::StateSyncBadRequest( + "offer_snapshot empty snapshot in request".to_string(), + ))?; + + tracing::debug!( + height = offered_snapshot.height, + version = offered_snapshot.version, + "[state_sync] api offer_snapshot", + ); + + // The grovedb wire version of the whole transfer is the OFFERED snapshot's version, + // validated against the single supported set. Unsupported versions ask Tenderdash to + // reject every snapshot of this format and try others. + let wire_version = u16::try_from(offered_snapshot.version) + .ok() + .filter(|version| SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS.contains(version)); + let Some(wire_version) = wire_version else { + tracing::warn!( + height = offered_snapshot.height, + version = offered_snapshot.version, + supported = ?SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, + "[state_sync] offer_snapshot rejecting unsupported snapshot version", + ); + return Ok(proto::ResponseOfferSnapshot { + result: response_offer_snapshot::Result::RejectFormat.into(), + }); + }; + + // The Platform version of the SNAPSHOT, which is what every grovedb call of this + // transfer must run under. It cannot come from `self.state`: a node that state syncs + // has no saved state, so its in-memory platform state is still at the initial protocol + // version and would hand grovedb the wrong (much older) version table than the one the + // serving node generated the chunks with. + // + // Only versions that write the reduced platform state can be restored at all, so + // anything below v15 is refused here rather than after a full transfer. + let snapshot_platform_version = decode_snapshot_metadata(&offered_snapshot.metadata) + .filter(|protocol_version| *protocol_version >= PROTOCOL_VERSION_15) + .and_then(|protocol_version| PlatformVersion::get(protocol_version).ok()); + let Some(snapshot_platform_version) = snapshot_platform_version else { + tracing::warn!( + height = offered_snapshot.height, + metadata = hex::encode(&offered_snapshot.metadata), + "[state_sync] offer_snapshot rejecting a snapshot without a usable platform version in its metadata", + ); + return Ok(proto::ResponseOfferSnapshot { + result: response_offer_snapshot::Result::Reject.into(), + }); + }; + + let mut session_write_guard = app.snapshot_fetching_session().write().map_err(|_| { + AbciError::StateSyncInternalError( + "offer_snapshot unable to lock session (poisoned)".to_string(), + ) + })?; + + if let Some(session) = session_write_guard.as_ref() { + // Every offer Tenderdash makes is Tenderdash resetting the transfer, so it always + // replaces the session in progress — including one for a LOWER height. + // + // The height in a snapshot descriptor is peer-supplied and untrusted (only the + // `app_hash` is light-client verified), so refusing to go backwards would hand a + // peer a wedge: advertise a high snapshot, withhold its chunks, and Tenderdash's + // fallback to an honest peer's older checkpoint would then be answered with an + // ABCI exception that aborts state sync altogether. Replacing is safe because the + // restore is only ever accepted against the verified app hash of whatever offer + // won. + tracing::warn!( + current_height = session.snapshot.height, + offered_height = offered_snapshot.height, + "[state_sync] offer_snapshot replacing session in progress", + ); + } + + // Mark the database as under restore BEFORE destroying it. From here until the + // restore completes, the node may be in a state that cannot serve consensus, and the + // only thing that can tell a restarted process so is this marker: without it, startup + // finds a database that disagrees with its platform state and cannot distinguish an + // interrupted restore from corruption. See `Platform::open_with_client`. + write_restore_sentinel( + &app.platform().config.db_path, + &request_app_hash, + offered_snapshot.height, + ) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "offer_snapshot unable to record the restore sentinel: {}", + e + )) + })?; + + // Both the fresh-session and the replace-session paths wipe grovedb (dropping the + // caches derived from it), start a new grovedb sync session, and answer Accept. + wipe_drive_for_restore(&app.platform().drive).map_err(|e| { + AbciError::StateSyncInternalError(format!("offer_snapshot unable to wipe grovedb: {}", e)) + })?; + + let state_sync_info = app + .platform() + .drive + .grove + .start_snapshot_syncing( + request_app_hash, + STATE_SYNC_SUBTREES_BATCH_SIZE, + wire_version, + &snapshot_platform_version.drive.grove_version, + ) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "offer_snapshot unable to start snapshot syncing session: {}", + e + )) + })?; + + *session_write_guard = Some(SnapshotFetchingSession { + snapshot: offered_snapshot, + app_hash: request_app_hash, + wire_version, + platform_version: snapshot_platform_version, + state_sync_info, + }); + + Ok(proto::ResponseOfferSnapshot { + result: response_offer_snapshot::Result::Accept.into(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::test::helpers::setup::TestPlatformBuilder; + + use crate::platform_types::snapshot::encode_snapshot_metadata; + + fn offer_at(height: u64, version: u32) -> proto::RequestOfferSnapshot { + offer_at_with_metadata( + height, + version, + encode_snapshot_metadata(PROTOCOL_VERSION_15), + ) + } + + fn offer_at_with_metadata( + height: u64, + version: u32, + metadata: Vec, + ) -> proto::RequestOfferSnapshot { + proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height, + version, + hash: vec![7u8; 32], + metadata, + }), + app_hash: vec![7u8; 32], + } + } + + /// The snapshot's Platform version drives every grovedb call of the transfer, so an + /// offer that does not carry a usable one must be refused BEFORE the database is + /// wiped — and refused as a per-snapshot Reject, so Tenderdash keeps walking its + /// ladder instead of aborting state sync. + #[test] + fn offer_snapshot_rejects_offers_without_a_usable_platform_version() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + for metadata in [ + vec![], // absent + vec![0u8; 3], // wrong length + encode_snapshot_metadata(1), // pre-v15, cannot be restored + encode_snapshot_metadata(u32::MAX), // unknown version + encode_snapshot_metadata(PROTOCOL_VERSION_15 - 1), // last version before v15 + ] { + let response = offer_snapshot(&app, offer_at_with_metadata(100, 1, metadata.clone())) + .expect("should not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Reject), + "metadata {:?} must be rejected", + metadata + ); + assert!( + app.snapshot_fetching_session.read().unwrap().is_none(), + "a rejected offer must not open a session", + ); + } + } + + #[test] + fn offer_snapshot_rejects_unsupported_version_with_reject_format() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + let response = offer_snapshot(&app, offer_at(100, 999)).expect("should not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::RejectFormat) + ); + assert!(app.snapshot_fetching_session.read().unwrap().is_none()); + } + + #[test] + fn offer_snapshot_accepts_fresh_and_replacing_offers() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + // Fresh session is accepted + let response = offer_snapshot(&app, offer_at(100, 1)).expect("should accept fresh offer"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + + // A LOWER height while syncing is Tenderdash falling back to another available + // snapshot after the higher one turned out to be unservable. It must replace the + // session and be accepted, otherwise a peer that advertises a high snapshot and + // then withholds its chunks could block the fallback. + let response = + offer_snapshot(&app, offer_at(50, 1)).expect("should accept an older fallback offer"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + assert_eq!( + app.snapshot_fetching_session + .read() + .unwrap() + .as_ref() + .expect("session must exist") + .snapshot + .height, + 50, + ); + + // Bring the session back up to 100 for the restart check below + offer_snapshot(&app, offer_at(100, 1)).expect("should accept offer"); + + // A same-height re-offer is a snapshot restart (Tenderdash RETRY_SNAPSHOT): + // the session is replaced and the offer accepted + let response = offer_snapshot(&app, offer_at(100, 1)).expect("should accept restart"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + + // A newer snapshot replaces the session and MUST also answer Accept + // (the old prototype returned the default UNKNOWN result here) + let response = offer_snapshot(&app, offer_at(200, 1)).expect("should accept newer offer"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + let session_guard = app.snapshot_fetching_session.read().unwrap(); + let session = session_guard.as_ref().expect("session must exist"); + assert_eq!(session.snapshot.height, 200); + assert_eq!(session.wire_version, 1); + assert_eq!( + session.platform_version.protocol_version, PROTOCOL_VERSION_15, + "the session must run under the SNAPSHOT's platform version", + ); + } +} diff --git a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs index e86a9ddb3eb..e7f9df5b222 100644 --- a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs @@ -8,6 +8,7 @@ use tenderdash_abci::proto::types::ConsensusParams; mod v0; mod v1; +mod v2; pub(crate) fn consensus_params_update( network: Network, @@ -33,9 +34,15 @@ pub(crate) fn consensus_params_update( new_platform_version, epoch_info, )), + 2 => Ok(v2::consensus_params_update_v2( + network, + original_platform_version, + new_platform_version, + epoch_info, + )), version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { method: "consensus_params_update".to_string(), - known_versions: vec![0, 1], + known_versions: vec![0, 1, 2], received: version, })), } @@ -143,7 +150,7 @@ mod tests { received, })) => { assert_eq!(method, "consensus_params_update"); - assert_eq!(known_versions, vec![0, 1]); + assert_eq!(known_versions, vec![0, 1, 2]); assert_eq!(received, 99); } other => panic!("expected UnknownVersionMismatch error, got: {:?}", other), @@ -587,4 +594,72 @@ mod tests { assert!(result.is_none()); } } + + mod v2_evidence_params { + use super::*; + + /// Crossing to v15 (whose method table selects consensus_params_update v2) must + /// emit both the new app version and the evidence params from issue #2512. + #[test] + fn crossing_to_v15_emits_evidence_params() { + let platform_v14 = PlatformVersion::get(14).expect("v14 exists"); + let platform_v15 = PlatformVersion::get(15).expect("v15 exists"); + let epoch_info = epoch_change_to(10); + + let params = + consensus_params_update(Network::Devnet, platform_v14, platform_v15, &epoch_info) + .expect("should not error") + .expect("crossing to v15 must emit consensus params"); + + let version = params.version.expect("version params must be set"); + assert_eq!(version.app_version, 15); + + let evidence = params.evidence.expect("evidence params must be set"); + assert_eq!(evidence.max_age_num_blocks, 15_000); + assert_eq!( + evidence + .max_age_duration + .expect("max age duration must be set") + .seconds, + 20 * 24 * 60 * 60 + ); + assert_eq!(evidence.max_bytes, 1_048_576); + } + + /// Once the network is on v15, a block without a version change emits nothing: + /// the evidence params are a one-shot emission on the activation block. + #[test] + fn steady_state_v15_emits_nothing() { + let platform_v15 = PlatformVersion::get(15).expect("v15 exists"); + let epoch_info = mid_epoch(11); + + let result = + consensus_params_update(Network::Devnet, platform_v15, platform_v15, &epoch_info) + .expect("should not error"); + assert!(result.is_none()); + } + + /// A version change that does not cross the v15 boundary must not attach + /// evidence params even when dispatched through v2. + #[test] + fn non_crossing_version_change_has_no_evidence_params() { + let platform_v13 = PlatformVersion::get(13).expect("v13 exists"); + let platform_v14 = PlatformVersion::get(14).expect("v14 exists"); + let epoch_info = epoch_change_to(9); + + let params = v2::consensus_params_update_v2( + Network::Devnet, + platform_v13, + platform_v14, + &epoch_info, + ) + .expect("version change must emit consensus params"); + + assert!(params.version.is_some()); + assert!( + params.evidence.is_none(), + "evidence params are only for the v15 crossing" + ); + } + } } diff --git a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs new file mode 100644 index 00000000000..b456b293593 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs @@ -0,0 +1,63 @@ +use crate::execution::engine::consensus_params_update::v1::consensus_params_update_v1; +use crate::platform_types::epoch_info::EpochInfo; +use dpp::dashcore::Network; +use dpp::version::v15::PROTOCOL_VERSION_15; +use dpp::version::PlatformVersion; +use tenderdash_abci::proto::google::protobuf::Duration; +use tenderdash_abci::proto::types::{ConsensusParams, EvidenceParams}; + +/// Maximum evidence age in blocks, applied when the network crosses to protocol +/// version 15 (state sync). Value proposed in issue #2512 for nodes that bootstrap +/// from snapshots and do not hold full history. +/// +/// REVIEW BEFORE RELEASE. Tenderdash treats evidence as expired only when BOTH bounds +/// are exceeded (`evidence/pool.go` `isExpired`), and the state sync backfill likewise +/// stops only once BOTH are satisfied (`statesync/reactor.go` `Backfill`). So the +/// effective window is the LARGER of the two: at ~6s blocks 15_000 blocks is about one +/// day and never binds, and a state-synced node backfills the full 20 days of +/// [`V15_EVIDENCE_MAX_AGE_DURATION_SECONDS`] below. If a ~1 day window was the intent +/// of #2512, the duration is the value to lower. Confirm before this ships. +const V15_EVIDENCE_MAX_AGE_NUM_BLOCKS: i64 = 15_000; + +/// Maximum evidence age in time: 20 days, per issue #2512. See the review note on +/// [`V15_EVIDENCE_MAX_AGE_NUM_BLOCKS`]. +const V15_EVIDENCE_MAX_AGE_DURATION_SECONDS: i64 = 20 * 24 * 60 * 60; + +/// Maximum total evidence per block in bytes. Tenderdash's default (1 MiB); #2512 does +/// not change it, but the whole evidence section must be populated when it is emitted. +const V15_EVIDENCE_MAX_BYTES: i64 = 1_048_576; + +/// Same as v1, but the first block of protocol version 15 additionally emits evidence +/// params sized for a network whose nodes may have bootstrapped via state sync +/// (issue #2512). +#[inline(always)] +pub(super) fn consensus_params_update_v2( + network: Network, + original_platform_version: &PlatformVersion, + new_platform_version: &PlatformVersion, + epoch_info: &EpochInfo, +) -> Option { + let mut consensus_params = consensus_params_update_v1( + network, + original_platform_version, + new_platform_version, + epoch_info, + )?; + + // Crossing to v15 implies a protocol version change, so v1 always emits params on + // the activation block and we only need to attach the evidence section. + let is_crossing_to_v15 = original_platform_version.protocol_version < PROTOCOL_VERSION_15 + && new_platform_version.protocol_version >= PROTOCOL_VERSION_15; + if is_crossing_to_v15 { + consensus_params.evidence = Some(EvidenceParams { + max_age_num_blocks: V15_EVIDENCE_MAX_AGE_NUM_BLOCKS, + max_age_duration: Some(Duration { + seconds: V15_EVIDENCE_MAX_AGE_DURATION_SECONDS, + nanos: 0, + }), + max_bytes: V15_EVIDENCE_MAX_BYTES, + }); + } + + Some(consensus_params) +} diff --git a/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs b/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs index e34fee97d36..fd74c21372d 100644 --- a/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs @@ -144,13 +144,28 @@ where self.config.execution.epoch_time_length_s, )?); + let mut consensus_params = consensus_params_update( + self.config.network, + first_platform_version, + platform_version, + &epoch_info, + )?; + + // `first_platform_version` above is a fiction for Tenderdash's benefit — it + // makes the update carry the real app version, since Tenderdash starts genesis + // assuming the first one. But it also makes a chain that STARTS on protocol + // v15+ look like it just crossed to v15, and the update then carries the + // evidence window override meant for chains upgrading with pre-state-sync + // genesis documents (#2512) — silently clobbering the evidence params of the + // genesis document being initialized right now. At genesis the operator's + // genesis document is authoritative, so the evidence section must not be + // emitted here; a `None` section leaves the genesis values in force. + if let Some(params) = consensus_params.as_mut() { + params.evidence = None; + } + Ok(ResponseInitChain { - consensus_params: consensus_params_update( - self.config.network, - first_platform_version, - platform_version, - &epoch_info, - )?, + consensus_params, app_hash: app_hash.to_vec(), validator_set_update: Some(validator_set), next_core_chain_lock_update: None, diff --git a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs index 68d87275ace..f0063a452f9 100644 --- a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs @@ -15,6 +15,7 @@ use dpp::version::PlatformVersion; use drive::grovedb::Transaction; mod v0; +mod v1; impl Platform where @@ -154,9 +155,19 @@ Your software version: {}, latest supported protocol version: {}."#, block_platform_version, timer, ), + 1 => self.run_block_proposal_v1( + block_proposal, + known_from_us, + epoch_info, + transaction, + platform_state, + block_platform_state, + block_platform_version, + timer, + ), version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { method: "run_block_proposal".to_string(), - known_versions: vec![0], + known_versions: vec![0, 1], received: version, })), } diff --git a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs new file mode 100644 index 00000000000..9ad0bf3f071 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs @@ -0,0 +1,507 @@ +use dpp::block::epoch::Epoch; + +use dpp::validation::ValidationResult; + +use dpp::version::PlatformVersion; +use drive::grovedb::Transaction; + +use crate::abci::AbciError; +use crate::error::execution::ExecutionError; + +use crate::error::Error; +use crate::execution::types::block_execution_context::v0::{ + BlockExecutionContextV0Getters, BlockExecutionContextV0MutableGetters, +}; +use crate::execution::types::block_execution_context::BlockExecutionContext; +use crate::execution::types::block_fees::v0::BlockFeesV0; +use crate::execution::types::block_state_info::v0::{ + BlockStateInfoV0Getters, BlockStateInfoV0Methods, BlockStateInfoV0Setters, +}; +use crate::execution::types::{block_execution_context, block_state_info}; +use crate::metrics::HistogramTiming; +use crate::platform_types::block_execution_outcome; +use crate::platform_types::block_proposal; +use crate::platform_types::epoch_info::v0::{EpochInfoV0Getters, EpochInfoV0Methods}; +use crate::platform_types::epoch_info::EpochInfo; +use crate::platform_types::platform::Platform; +use crate::platform_types::platform_state::PlatformState; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::verify_chain_lock_result::v0::VerifyChainLockResult; +use crate::rpc::core::CoreRPCLike; +use dpp::reduced_platform_state::v0::ReducedBlockInfoV0; + +impl Platform +where + C: CoreRPCLike, +{ + /// Runs a block proposal, either from process proposal or prepare proposal. + /// + /// This function takes a `BlockProposal` and a `Transaction` as input and processes the block + /// proposal. It first validates the block proposal and then processes raw state transitions, + /// withdrawal transactions, and block fees. It also updates the validator set. + /// + /// v1 (protocol v15, state sync): identical to v0 except that + /// `validator_set_update` runs BEFORE the root hash is computed (it only mutates the + /// in-memory block platform state, never grovedb, so the move cannot change the root + /// hash or the rotation outcome), and the reduced platform state — including the + /// post-rotation next validator set — is then written into the replicated grovedb + /// state immediately before the root hash, so it is covered by this block's app hash + /// and a state-synced node can reconstruct the full platform state from it. + /// + /// # Arguments + /// + /// * `block_proposal` - The block proposal to be processed. + /// * `known_from_us` - Do we know that we made this block proposal?. + /// * `transaction` - The transaction associated with the block proposal. + /// + /// # Returns + /// + /// * `Result, Error>` - If the block proposal is + /// successfully processed, it returns a `ValidationResult` containing the `BlockExecutionOutcome`. + /// If the block proposal processing fails, it returns an `Error`. Consensus errors are returned + /// in the `ValidationResult`, while critical system errors are returned in the `Result`. + /// + /// # Errors + /// + /// This function may return an `Error` variant if there is a problem with processing the block + /// proposal, updating the core info, processing raw state transitions, or processing block fees. + /// + #[allow(clippy::too_many_arguments)] + pub(super) fn run_block_proposal_v1( + &self, + block_proposal: block_proposal::v0::BlockProposal, + known_from_us: bool, + epoch_info: EpochInfo, + transaction: &Transaction, + last_committed_platform_state: &PlatformState, + mut block_platform_state: PlatformState, + platform_version: &'static PlatformVersion, + timer: Option<&HistogramTiming>, + ) -> Result, Error> + { + tracing::trace!( + method = "run_block_proposal_v1", + ?block_proposal, + ?epoch_info, + "Running a block proposal for height: {}, round: {}", + block_proposal.height, + block_proposal.round, + ); + + // Run block proposal determines version by itself based on the previous + // state and block time. + // It should provide correct version on prepare proposal to block header + // and validate it on process proposal. + // If version set to 0 (default number value) it means we are on prepare proposal, + // so there is no need for validation. + if !known_from_us + && block_proposal.consensus_versions.app != platform_version.protocol_version as u64 + { + return Ok(ValidationResult::new_with_error( + AbciError::BadRequest(format!( + "received a block proposal with protocol version {}, expected: {}", + block_proposal.consensus_versions.app, platform_version.protocol_version + )) + .into(), + )); + } + + let last_block_time_ms = last_committed_platform_state.last_committed_block_time_ms(); + let last_block_height = last_committed_platform_state.last_committed_known_block_height_or( + self.config.abci.genesis_height.saturating_sub(1), + ); + let last_block_core_height = last_committed_platform_state + .last_committed_known_core_height_or(self.config.abci.genesis_core_height); + + // Init block execution context + let block_state_info = block_state_info::v0::BlockStateInfoV0::from_block_proposal( + &block_proposal, + last_block_time_ms, + ); + + // First let's check that this is the follower to a previous block + if !block_state_info.next_block_to(last_block_height, last_block_core_height)? { + // we are on the wrong height or round + return Ok(ValidationResult::new_with_error(AbciError::WrongBlockReceived(format!( + "received a block proposal for height: {} core height: {}, current height: {} core height: {}", + block_state_info.height, block_state_info.core_chain_locked_height, last_block_height, last_block_core_height + )).into())); + } + + // destructure the block proposal + let block_proposal::v0::BlockProposal { + core_chain_locked_height, + core_chain_lock_update, + proposed_app_version, + proposer_pro_tx_hash, + validator_set_quorum_hash, + raw_state_transitions, + .. + } = block_proposal; + + let block_info = block_state_info.to_block_info( + Epoch::new(epoch_info.current_epoch_index()) + .expect("current epoch index should be in range"), + ); + + if epoch_info.is_epoch_change_but_not_genesis() { + tracing::info!( + epoch_index = epoch_info.current_epoch_index(), + "epoch change occurring from epoch {} to epoch {}", + epoch_info + .previous_epoch_index() + .expect("must be set since we aren't on genesis"), + epoch_info.current_epoch_index(), + ); + } + + // Update block platform state with current and next epoch protocol versions + // if it was proposed + // This is happening only on epoch change + self.upgrade_protocol_version_on_epoch_change( + &block_info, + &epoch_info, + last_committed_platform_state, + &mut block_platform_state, + transaction, + platform_version, + )?; + + // If there is a core chain lock update, we should start by verifying it + if let Some(core_chain_lock_update) = core_chain_lock_update.as_ref() { + if !known_from_us { + let verification_result = self.verify_chain_lock( + block_state_info.round, // the round is to allow us to bypass local verification in case of chain stall + &block_platform_state, + core_chain_lock_update, + true, // if it's not known from us, then we should try submitting it + platform_version, + ); + + let VerifyChainLockResult { + chain_lock_signature_is_deserializable, + found_valid_locally, + found_valid_by_core, + core_is_synced, + } = match verification_result { + Ok(verification_result) => verification_result, + Err(Error::Execution(e)) => { + // This will happen only if an internal version error + return Err(Error::Execution(e)); + } + Err(e) => { + // This will happen only if a core rpc error + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(e.to_string()).into(), + )); + } + }; + + if !chain_lock_signature_is_deserializable { + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(format!( + "received a chain lock for height {} that has a signature that can not be deserialized {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + + if let Some(found_valid_locally) = found_valid_locally { + // This means we are able to check if the chain lock is valid + if !found_valid_locally { + // The signature was not valid + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(format!( + "received a chain lock for height {} that we figured out was invalid based on platform state {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + } + + if let Some(found_valid_by_core) = found_valid_by_core { + // This means we asked core if the chain lock was valid + if !found_valid_by_core { + // Core said it wasn't valid + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(format!( + "received a chain lock for height {} that is invalid based on a core request {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + } + + if let Some(core_is_synced) = core_is_synced { + // Core is just not synced + if !core_is_synced { + // The submission was not accepted by core + return Ok(ValidationResult::new_with_error( + AbciError::ChainLockedBlockNotKnownByCore(format!( + "received a chain lock for height {} that we could not accept because core is not synced {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + } + } + } + + // Update the masternode list and create masternode identities and also update the active quorums + self.update_core_info( + Some(last_committed_platform_state), + &mut block_platform_state, + core_chain_locked_height, + false, + &block_info, + transaction, + platform_version, + )?; + + // Update the validator proposed app version + // It should be called after protocol version upgrade + self.drive + .update_validator_proposed_app_version( + proposer_pro_tx_hash, + proposed_app_version as u32, + Some(transaction), + &platform_version.drive, + ) + .map_err(|e| { + Error::Execution(ExecutionError::UpdateValidatorProposedAppVersionError(e)) + })?; // This is a system error + + // Rebroadcast expired withdrawals if they exist + // We do that before we mark withdrawals as expired + // to rebroadcast them on the next block but not the same + // one + // TODO: It must be also only on core height change + self.rebroadcast_expired_withdrawal_documents( + &block_info, + last_committed_platform_state, + transaction, + platform_version, + )?; + + // Mark all previously broadcasted and chainlocked withdrawals as complete + // only when we are on a new core height + if block_state_info.core_chain_locked_height() != last_block_core_height { + self.update_broadcasted_withdrawal_statuses( + &block_info, + transaction, + platform_version, + )?; + } + + // Preparing withdrawal transactions for signing and broadcasting + // To process withdrawals we need to dequeue untiled transactions from the withdrawal transactions queue + // Untiled transactions then converted to unsigned transactions, appending current block information + // required for signature verification (core height and quorum hash) + // Then we save unsigned transaction bytes to block execution context + // to be signed (on extend_vote), verified (on verify_vote) and broadcasted (on finalize_block) + // Also, the dequeued untiled transaction added to the broadcasted transaction queue to for further + // resigning in case of failures. + let unsigned_withdrawal_transaction_bytes = self + .dequeue_and_build_unsigned_withdrawal_transactions( + validator_set_quorum_hash, + &block_info, + Some(transaction), + platform_version, + )?; + + // Run all dao platform events, such as vote tallying and distribution of contested documents + // This must be done before state transition processing + // Otherwise we would expect a proof after a successful vote that has since been cleaned up. + self.run_dao_platform_events( + &block_info, + last_committed_platform_state, + &block_platform_state, + Some(transaction), + platform_version, + )?; + + // Process transactions + let state_transitions_result = self.process_raw_state_transitions( + raw_state_transitions, + &block_platform_state, + &block_info, + transaction, + platform_version, + known_from_us, + timer, + )?; + + // Store the address balances to recent block storage + self.store_address_balances_to_recent_block_storage( + &state_transitions_result.address_balances_updated, + &block_info, + transaction, + platform_version, + )?; + + // Clean up expired compacted address balance entries + self.cleanup_recent_block_storage_address_balances( + &block_info, + transaction, + platform_version, + )?; + + // Record shielded pool anchor if the commitment tree changed this block. + // This stores block_height → anchor_bytes so shielded transactions can + // reference a recent anchor for spend authorization. + self.record_shielded_pool_anchor_if_changed( + block_proposal.height, + transaction, + platform_version, + )?; + + // Prune anchors older than the configured retention depth + self.prune_shielded_pool_anchors(block_proposal.height, transaction, platform_version)?; + + // Pool withdrawals into transactions queue + + // Takes queued withdrawals, creates untiled withdrawal transaction payload, saves them to queue + // Corresponding withdrawal documents are changed from queued to pooled + self.pool_withdrawals_into_transactions_queue( + &block_info, + last_committed_platform_state, + Some(transaction), + platform_version, + )?; + + // Cleans up the expired locks for withdrawal amounts + // to update daily withdrawal limit + // This is for example when we make a withdrawal for 30 Dash + // But we can only withdraw 1000 Dash a day + // after the withdrawal we should only be able to withdraw 970 Dash + // But 24 hours later that locked 30 comes back + self.clean_up_expired_locks_of_withdrawal_amounts( + &block_info, + transaction, + platform_version, + )?; + + // Create a new block execution context + + let mut block_execution_context: BlockExecutionContext = + block_execution_context::v0::BlockExecutionContextV0 { + block_state_info: block_state_info.into(), + epoch_info, + unsigned_withdrawal_transactions: unsigned_withdrawal_transaction_bytes, + block_address_balance_changes: std::collections::BTreeMap::new(), + block_platform_state, + proposer_results: None, + } + .into(); + + // while we have the state transitions executed, we now need to process the block fees + let block_fees_v0: BlockFeesV0 = state_transitions_result.aggregated_fees().clone().into(); + + // Process fees + let processed_block_fees = self.process_block_fees_and_validate_sum_trees( + &block_execution_context, + block_fees_v0.into(), + transaction, + platform_version, + )?; + + tracing::debug!(block_fees = ?processed_block_fees, "block fees are processed"); + + // Record the credits this block minted into Platform (asset locks funding state + // transitions, epoch Core rewards) as a credit inflow: the daily withdrawal limit adds + // inflows younger than its day-old base to the daily maximum, so it limits net outflow. + // A system event, so nobody pays fees for the write. + self.record_credit_inflows_for_withdrawals( + state_transitions_result + .credit_mints() + .saturating_add(processed_block_fees.credit_mints), + &block_info, + transaction, + platform_version, + )?; + + // Record the total credits in Platform if this block changed it: the daily withdrawal + // limit is a share of the total credits Platform held a day ago, read from this history. + // This runs after fees and epoch rewards, the last things in a block that can move the + // total, and before the app hash so the entry is part of this block's state. + self.record_total_credits_history_for_withdrawals( + &block_info, + transaction, + platform_version, + )?; + + // Unlike v0, the validator set update happens BEFORE the root hash is computed. + // It only mutates the in-memory block platform state (the rotated + // next_validator_set_quorum_hash) and never touches grovedb, so the rotation + // outcome and the root hash are unaffected by the move; it must come first so + // the reduced platform state written below carries the post-rotation state. + let validator_set_update = self.validator_set_update( + block_proposal.proposer_pro_tx_hash, + last_committed_platform_state, + &mut block_execution_context, + platform_version, + )?; + + // Write the reduced platform state into the replicated grovedb state, immediately + // before the root hash so it is covered by this block's app hash. A state-synced + // node reads it back to reconstruct the full platform state, which otherwise only + // exists in non-replicated aux storage. Only header-fixed fields go in: the same + // block re-proposed at a later round must hash to the same app hash, so the round + // (and the not-yet-known app hash, block id hash and signature) stay out. + let reduced_platform_state = block_execution_context + .block_platform_state() + .to_reduced_platform_state( + Some(ReducedBlockInfoV0 { + basic_info: block_info, + quorum_hash: validator_set_quorum_hash.into(), + proposer_pro_tx_hash: proposer_pro_tx_hash.into(), + }), + core_chain_locked_height, + ); + + self.store_reduced_platform_state( + &reduced_platform_state, + Some(transaction), + platform_version, + )?; + + let root_hash = self + .drive + .grove + .root_hash(Some(transaction), &platform_version.drive.grove_version) + .unwrap() + .map_err(|e| Error::Drive(drive::error::Error::from(e)))?; //GroveDb errors are system errors + + block_execution_context + .block_state_info_mut() + .set_app_hash(Some(root_hash)); + + if tracing::enabled!(tracing::Level::TRACE) { + tracing::trace!( + method = "run_block_proposal_v1", + app_hash = hex::encode(root_hash), + block_hash = hex::encode(block_proposal.block_hash.unwrap_or_default()), + platform_state_fingerprint = hex::encode( + block_execution_context + .block_platform_state() + .fingerprint()? + ), + "Block proposal executed successfully", + ); + } + + Ok(ValidationResult::new_with_data( + block_execution_outcome::v0::BlockExecutionOutcome { + app_hash: root_hash, + state_transitions_result, + validator_set_update, + platform_version, + block_execution_context, + }, + )) + } +} diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs index c6b1d43549e..632c68e4e58 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs @@ -37,13 +37,19 @@ where let block_height = platform_state.last_committed_block_height(); let block_time = platform_state.last_committed_block_time_ms().unwrap_or(0); - let keep_n = platform_version.drive_abci.checkpoints.num_checkpoints as usize; - - // Build the checkpoint path: db_path/checkpoints/ - let checkpoint_path = self - .config - .db_path - .join("checkpoints") + // When snapshot serving is enabled, the operator-provided state sync + // configuration overrides the platform-version-driven checkpoint retention. + let state_sync_config = &self.config.abci.state_sync; + let keep_n = if state_sync_config.snapshots_enabled { + state_sync_config.max_num_snapshots + } else { + platform_version.drive_abci.checkpoints.num_checkpoints as usize + }; + + // Build the checkpoint path: / + // (defaults to db_path/checkpoints) + let checkpoint_path = state_sync_config + .resolved_checkpoints_path(&self.config.db_path) .join(block_height.to_string()); // Create the parent checkpoints directory if it doesn't exist diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs index 79d2034f99c..45d8412b7f6 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs @@ -39,10 +39,22 @@ where return Ok(None); } - // How often we want a checkpoint - let checkpoint_interval_milliseconds = - platform_version.drive_abci.checkpoints.frequency_seconds as u64 * 1000; - let keep_n = platform_version.drive_abci.checkpoints.num_checkpoints as usize; + // How often we want a checkpoint. When snapshot serving is enabled, the + // operator-provided state sync configuration overrides the + // platform-version-driven checkpoint parameters. + let state_sync_config = &self.config.abci.state_sync; + let (frequency_seconds, keep_n) = if state_sync_config.snapshots_enabled { + ( + state_sync_config.snapshots_frequency_seconds as u64, + state_sync_config.max_num_snapshots, + ) + } else { + ( + platform_version.drive_abci.checkpoints.frequency_seconds as u64, + platform_version.drive_abci.checkpoints.num_checkpoints as usize, + ) + }; + let checkpoint_interval_milliseconds = frequency_seconds * 1000; // If disabled or misconfigured, do nothing. if checkpoint_interval_milliseconds == 0 || keep_n == 0 { diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs index f3b297ce7e0..a75d735c551 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs @@ -33,13 +33,19 @@ where return Ok(false); }; - let keep_n = platform_version.drive_abci.checkpoints.num_checkpoints as usize; + // When snapshot serving is enabled, the operator-provided state sync + // configuration overrides the platform-version-driven checkpoint retention. + let state_sync_config = &self.config.abci.state_sync; + let keep_n = if state_sync_config.snapshots_enabled { + state_sync_config.max_num_snapshots + } else { + platform_version.drive_abci.checkpoints.num_checkpoints as usize + }; - // Build the checkpoint path: db_path/checkpoints/ - let checkpoint_path = self - .config - .db_path - .join("checkpoints") + // Build the checkpoint path: / + // (defaults to db_path/checkpoints) + let checkpoint_path = state_sync_config + .resolved_checkpoints_path(&self.config.db_path) .join(block_height.to_string()); // Create the checkpoints directory if it doesn't exist diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs index 62f1cdfab8c..e40f0adfafe 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs @@ -971,5 +971,128 @@ mod tests { "wrap-around should not trigger when last block was on different quorum" ); } + + /// run_block_proposal v1 (protocol v15) moves `validator_set_update` from AFTER + /// the root-hash computation (its v0 position) to BEFORE it, so the reduced + /// platform state written into the replicated state can carry the post-rotation + /// next validator set. The only observable differences between the two call + /// sites are (a) `block_state_info.app_hash` being set and (b) grovedb having + /// received additional writes in between. Rotation reads neither, and this test + /// proves it: for rotation-triggering and non-triggering scenarios alike, the + /// rotation outcome (returned update and resulting next validator set quorum + /// hash) is identical whether or not the app hash was set and grovedb was + /// written to before the call. + #[test] + fn v2_rotation_outcome_is_independent_of_root_hash_ordering() { + use crate::execution::types::block_execution_context::v0::BlockExecutionContextV0MutableGetters; + use crate::execution::types::block_state_info::v0::BlockStateInfoV0Setters; + use dpp::reduced_platform_state::v0::ReducedBlockInfoV0; + + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + + let mut rng = StdRng::seed_from_u64(57); + let qh1 = quorum_hash_from_seed(1); + let qh2 = quorum_hash_from_seed(2); + let vs1 = make_validator_set(qh1, &[10, 20, 30], &mut rng); + let vs2 = make_validator_set(qh2, &[40, 50, 60], &mut rng); + + let mut validator_sets = IndexMap::new(); + validator_sets.insert(qh1, vs1); + validator_sets.insert(qh2, vs2); + + // Scenarios: (proposer seed, last committed proposer seed, description) + // - proposer 20 after 10: mid-quorum, no rotation + // - proposer 30 after 20: last member, rotation to qh2 + // - proposer 10 after 20: wrap-around, rotation to qh2 + let scenarios: [(u8, u8, &str); 3] = [ + (20, 10, "no rotation"), + (30, 20, "rotation on last member"), + (10, 20, "rotation on wrap-around"), + ]; + + for (proposer_seed, last_proposer_seed, description) in scenarios { + let mut platform_state = platform.state.load().as_ref().clone(); + platform_state.set_current_validator_set_quorum_hash(qh1); + platform_state.set_validator_sets(validator_sets.clone()); + let mut last_proposer = [0u8; 32]; + last_proposer[31] = last_proposer_seed; + platform_state.set_last_committed_block_info(Some(make_extended_block_info( + *qh1.as_byte_array(), + last_proposer, + 5, + ))); + + let mut proposer = [0u8; 32]; + proposer[31] = proposer_seed; + + // v1 ordering: rotation runs before the root hash exists and before any + // reduced-state write. + let mut context_before_root_hash = + make_block_execution_context(platform_state.clone()); + let update_before = platform + .validator_set_update_v2( + proposer, + &platform_state, + &mut context_before_root_hash, + ) + .expect("should succeed before root hash"); + + // v0 ordering: by the time rotation runs, the app hash has been computed + // and set, and grovedb has received the block's writes (simulated here by + // a committed reduced-state write). + let reduced_platform_state = platform_state.to_reduced_platform_state( + Some(ReducedBlockInfoV0 { + basic_info: BlockInfo::default(), + quorum_hash: (*qh1.as_byte_array()).into(), + proposer_pro_tx_hash: proposer.into(), + }), + 1, + ); + platform + .store_reduced_platform_state(&reduced_platform_state, None, platform_version) + .expect("should store reduced platform state"); + let mut context_after_root_hash = + make_block_execution_context(platform_state.clone()); + context_after_root_hash + .block_state_info_mut() + .set_app_hash(Some([9u8; 32])); + let update_after = platform + .validator_set_update_v2( + proposer, + &platform_state, + &mut context_after_root_hash, + ) + .expect("should succeed after root hash"); + + assert_eq!( + update_before, update_after, + "validator set update must not depend on call ordering ({})", + description + ); + assert_eq!( + context_before_root_hash + .block_platform_state() + .next_validator_set_quorum_hash(), + context_after_root_hash + .block_platform_state() + .next_validator_set_quorum_hash(), + "next validator set quorum hash must not depend on call ordering ({})", + description + ); + assert_eq!( + context_before_root_hash + .block_platform_state() + .current_validator_set_quorum_hash(), + context_after_root_hash + .block_platform_state() + .current_validator_set_quorum_hash(), + "current validator set quorum hash must not depend on call ordering ({})", + description + ); + } + } } } diff --git a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/mod.rs index 1196d4ebe92..55a16d6118e 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/mod.rs @@ -66,4 +66,22 @@ where })), } } + + /// Rebuilds the in-memory Core-derived state (masternode lists and every quorum + /// set) from scratch at `core_block_height`, without touching GroveDB. + /// + /// State sync reconstruction uses this: the restored GroveDB already holds every + /// masternode identity exactly as the source chain wrote it, so the identity writes + /// `update_core_info` would issue are at best no-ops and at worst a root-hash + /// mismatch. Only the platform state, which is not replicated, has to be rebuilt. + /// Not consensus code, hence unversioned. + pub(crate) fn rebuild_core_info_in_memory( + &self, + state: &mut PlatformState, + core_block_height: u32, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + self.update_state_masternode_list_v0(state, core_block_height, true)?; + self.update_quorum_info(None, state, core_block_height, true, platform_version) + } } diff --git a/packages/rs-drive-abci/src/execution/platform_events/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/mod.rs index 1ac0715b9d2..32461d6a15e 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/mod.rs @@ -20,6 +20,8 @@ pub(in crate::execution) mod fee_pool_outwards_distribution; pub(in crate::execution) mod initialization; /// Protocol upgrade events pub(in crate::execution) mod protocol_upgrade; +/// State sync platform state reconstruction +pub(in crate::execution) mod state_sync; /// State transition processing pub(in crate::execution) mod state_transition_processing; mod tokens; diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs new file mode 100644 index 00000000000..f48dae65527 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs @@ -0,0 +1,3 @@ +//! State sync events: reconstruction of the platform state after a snapshot restore. + +mod reconstruct_platform_state; diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs new file mode 100644 index 00000000000..c2eb2e961a4 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs @@ -0,0 +1,440 @@ +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use crate::platform_types::platform_state::{PlatformState, PlatformStateV0Methods}; +use crate::platform_types::signature_verification_quorum_set::{ + Quorums, SignatureVerificationQuorumSet, SignatureVerificationQuorumSetV0Methods, + VerificationQuorum, +}; +use crate::platform_types::validator_set::ValidatorSet; +use crate::rpc::core::CoreRPCLike; +use dpp::block::extended_block_info::v0::ExtendedBlockInfoV0; +use dpp::block::extended_block_info::ExtendedBlockInfo; +use dpp::bls_signatures::PublicKey as BlsPublicKey; +use dpp::dashcore::hashes::Hash; +use dpp::dashcore::QuorumHash; +use dpp::fee::default_costs::CachedEpochIndexFeeVersions; +use dpp::platform_value::Bytes32; +use dpp::reduced_platform_state::v0::ReducedPreviousQuorumsV0; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::version::fee::FeeVersion; +use dpp::version::PlatformVersion; +use indexmap::IndexMap; +use std::collections::{BTreeMap, BTreeSet}; + +impl Platform +where + C: CoreRPCLike, +{ + /// Reconstructs the full in-memory platform state after a state sync snapshot + /// restore, and persists it to aux storage so it survives restarts. + /// + /// ## Expected state + /// + /// The restored grovedb contains the reduced platform state that + /// `run_block_proposal` v1 wrote while processing the snapshot block, i.e. the + /// state after the whole block including `validator_set_update`, immediately + /// before the root hash was computed. Reconstruction: + /// + /// 1. restores the scalar fields (protocol versions, quorum hashes, fee versions) + /// directly from the reduced state; + /// 2. re-derives the masternode lists and quorums from Core, in memory only, via + /// `rebuild_core_info_in_memory`; the restored grovedb already holds every + /// masternode identity, so nothing is written and the root hash cannot change; + /// 3. restores the validator set order recorded by the source (`quorum_positions`), + /// which cannot be recovered from Core RPC; + /// 4. advances the state to the snapshot block via `update_state_cache`, which + /// performs the same next-into-current validator set rotation the source node + /// performed when it finalized that block, persists the state to aux storage and + /// publishes it, so the `info` handler reports the snapshot height and app hash + /// after both this restore and any later restart. + pub fn reconstruct_platform_state( + &self, + app_hash: &[u8; 32], + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + let reduced_platform_state = self + .fetch_reduced_platform_state(None, platform_version)? + .ok_or_else(|| { + AbciError::StateSyncInternalError( + "reconstruct_platform_state restored snapshot does not contain a reduced \ + platform state (was it taken before the v15 activation height?)" + .to_string(), + ) + })?; + let ReducedPlatformState::V0(saved) = reduced_platform_state; + + // Everything below runs with the platform version the snapshot's chain was + // actually on, which may lag the version this binary considers latest. + let state_platform_version = + PlatformVersion::get(saved.current_protocol_version_in_consensus)?; + + // Restore the fee versions of previous epochs faithfully, by version number + let previous_fee_versions: CachedEpochIndexFeeVersions = saved + .previous_fee_versions + .iter() + .map(|(epoch_index, fee_version_number)| { + Ok((*epoch_index, FeeVersion::get(*fee_version_number)?)) + }) + .collect::>()?; + + let mut platform_state = PlatformState { + genesis_block_info: None, + last_committed_block_info: None, + current_protocol_version_in_consensus: saved.current_protocol_version_in_consensus, + next_epoch_protocol_version: saved.next_epoch_protocol_version, + current_validator_set_quorum_hash: QuorumHash::from_byte_array( + saved.current_validator_set_quorum_hash.to_buffer(), + ), + next_validator_set_quorum_hash: saved + .next_validator_set_quorum_hash + .map(|quorum_hash| QuorumHash::from_byte_array(quorum_hash.to_buffer())), + validator_sets: Default::default(), + chain_lock_validating_quorums: SignatureVerificationQuorumSet::new( + &self.config.chain_lock, + state_platform_version, + )?, + instant_lock_validating_quorums: SignatureVerificationQuorumSet::new( + &self.config.instant_lock, + state_platform_version, + )?, + full_masternode_list: Default::default(), + hpmn_masternode_list: Default::default(), + previous_fee_versions, + }; + + let saved_block_info = + saved + .last_committed_block_info + .ok_or(AbciError::StateSyncInternalError( + "reconstruct_platform_state reduced platform state has no last committed \ + block info" + .to_string(), + ))?; + + let current_block_info: ExtendedBlockInfo = ExtendedBlockInfoV0 { + basic_info: saved_block_info.basic_info, + app_hash: *app_hash, + quorum_hash: saved_block_info.quorum_hash.to_buffer(), + proposer_pro_tx_hash: saved_block_info.proposer_pro_tx_hash.to_buffer(), + // The block id hash, signature and round are not part of the reduced state + // (they are unknown while the block executes, and the round must not affect + // the app hash). They are zero until the next block is finalized; proofs are + // refused until then, see `ensure_block_proof_metadata_is_available`. + block_id_hash: [0u8; 32], + signature: [0u8; 96], + round: 0, + } + .into(); + + // Rebuild the Core-derived state in memory only, from scratch, at the core height + // the snapshot block ran with. The restored grovedb already + // holds every masternode identity as the source chain wrote it; rewriting them + // here would be thousands of no-op writes at best and a root-hash mismatch at + // worst, and the caller compares the root hash against the snapshot afterwards. + self.rebuild_core_info_in_memory( + &mut platform_state, + saved.proposed_core_chain_locked_height, + state_platform_version, + )?; + + // The validator sets live in the platform state, NOT in grovedb, so the caller's + // app-hash equality check cannot see a disagreement between what Core just handed + // us and what the snapshot source actually ran with. `quorum_positions` is the + // consensus-covered list of validator set hashes from the source, so require an + // exact match before publishing anything: a restored node running with validator + // sets the chain never agreed on is worse than no restore at all, and the caller + // turns this error into a REJECT_SNAPSHOT. + let derived_validator_sets: BTreeSet<[u8; 32]> = platform_state + .validator_sets() + .keys() + .map(|quorum_hash| quorum_hash.to_byte_array()) + .collect(); + let saved_validator_sets: BTreeSet<[u8; 32]> = saved + .quorum_positions + .iter() + .map(|quorum_hash| quorum_hash.to_buffer()) + .collect(); + if derived_validator_sets != saved_validator_sets { + return Err(AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state validator sets re-derived from Core do not match the \ + snapshot: {} derived, {} saved, {} only in Core, {} only in the snapshot", + derived_validator_sets.len(), + saved_validator_sets.len(), + derived_validator_sets + .difference(&saved_validator_sets) + .count(), + saved_validator_sets + .difference(&derived_validator_sets) + .count(), + )) + .into()); + } + + // Core RPC returns quorums in an order that need not match the incremental + // order the source node maintained; restore the recorded order. + sort_validator_sets_by_saved_positions( + platform_state.validator_sets_mut(), + &saved.quorum_positions, + ); + + // Reinstate the signature-verification quorum HISTORY. The rebuild above + // derived the current sets from Core — which is exact, the quorums of a type at a + // core height are whatever Core reports — but it was given `platform_state = None` + // and so could not produce any previous set. That history is consensus-relevant: + // `select_quorums` uses the previous set for locks signed within `SIGN_OFFSET` + // core blocks of a change, and for instant locks there is no Core fallback, so a + // restored node missing it would reject an asset lock proof the network accepted. + restore_previous_quorums( + platform_state.chain_lock_validating_quorums_mut(), + saved.previous_chain_lock_quorums.as_ref(), + )?; + restore_previous_quorums( + platform_state.instant_lock_validating_quorums_mut(), + saved.previous_instant_lock_quorums.as_ref(), + )?; + + let block_height = saved_block_info.basic_info.height; + + // Advance the state to the snapshot block: rotates next-into-current exactly as + // the source did on finalization, persists to aux storage and publishes the + // state for the info handler. Aux writes are not part of the root hash, so + // committing them separately cannot change the app hash the caller verifies. + let aux_transaction = self.drive.grove.start_transaction(); + self.update_state_cache( + current_block_info, + platform_state, + &aux_transaction, + state_platform_version, + )?; + self.drive + .grove + .commit_transaction(aux_transaction) + .unwrap() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state unable to commit aux transaction: {}", + e + )) + })?; + + tracing::debug!( + block_height, + app_hash = hex::encode(app_hash), + "[state_sync] platform state reconstructed", + ); + + Ok(()) + } +} + +/// Reinstates the superseded quorums of a signature-verification quorum set exactly as the +/// snapshot source held them. +/// +/// `None` is a legitimate answer (the source had seen no quorum change yet) and leaves the +/// set without a history, which is what the source had. +fn restore_previous_quorums( + quorum_set: &mut SignatureVerificationQuorumSet, + saved: Option<&ReducedPreviousQuorumsV0>, +) -> Result<(), Error> { + let Some(saved) = saved else { + return Ok(()); + }; + + let quorums = saved + .quorums + .iter() + .map(|quorum| { + let public_key = BlsPublicKey::try_from(quorum.public_key.as_slice()).map_err(|e| { + AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state previous quorum {} has an undeserializable public \ + key: {}", + hex::encode(quorum.quorum_hash.to_buffer()), + e + )) + })?; + + Ok(( + QuorumHash::from_byte_array(quorum.quorum_hash.to_buffer()), + VerificationQuorum { + public_key, + index: quorum.index, + }, + )) + }) + .collect::, Error>>()?; + + quorum_set.restore_previous_past_quorums( + quorums, + saved.last_active_core_height, + saved.updated_at_core_height, + saved.previous_change_height, + ); + + Ok(()) +} + +/// Sorts the validator sets into the order recorded in the reduced platform state. +/// +/// Validator sets not present in the recorded order (which should not happen when the +/// reduced state and Core agree on the quorum list) sort last, preserving their +/// relative order. +fn sort_validator_sets_by_saved_positions( + validator_sets: &mut IndexMap, + quorum_positions: &[Bytes32], +) { + let lookup_table: BTreeMap<&[u8], usize> = quorum_positions + .iter() + .enumerate() + .map(|(position, quorum_hash)| (quorum_hash.as_slice(), position)) + .collect(); + + validator_sets.sort_by(|a_hash, _, b_hash, _| { + let a_position = lookup_table + .get(a_hash.as_byte_array().as_slice()) + .unwrap_or(&usize::MAX); + let b_position = lookup_table + .get(b_hash.as_byte_array().as_slice()) + .unwrap_or(&usize::MAX); + + a_position.cmp(b_position) + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn quorum_hash(seed: u8) -> QuorumHash { + let mut bytes = [0u8; 32]; + bytes[31] = seed; + QuorumHash::from_byte_array(bytes) + } + + /// The quorum-set history that travels with a snapshot must come back byte for byte, + /// including `previous_change_height` — `set_previous_past_quorums` DERIVES that field + /// from whatever the set already holds, which on a freshly reconstructed set is + /// nothing, so restoring through it would silently lose it and change which quorums + /// `select_quorums` considers verifiable. + #[test] + fn should_restore_the_previous_quorum_history_verbatim() { + use crate::config::ChainLockConfig; + use crate::platform_types::platform_state::to_reduced_previous_quorums; + use dpp::bls_signatures::{Bls12381G2Impl, SecretKey}; + use rand::rngs::StdRng; + use rand::SeedableRng; + + let mut rng = StdRng::seed_from_u64(11); + let quorums: Quorums = [(1u8, None), (2u8, Some(3u32))] + .into_iter() + .map(|(seed, index)| { + ( + quorum_hash(seed), + VerificationQuorum { + public_key: SecretKey::::random(&mut rng).public_key(), + index, + }, + ) + }) + .collect(); + + let mut source = SignatureVerificationQuorumSet::new( + &ChainLockConfig::default_100_67(), + PlatformVersion::latest(), + ) + .expect("should build quorum set"); + // Two changes, so `previous_change_height` is populated and can be lost + source.set_previous_past_quorums(quorums.clone(), 900, 950); + source.set_previous_past_quorums(quorums.clone(), 990, 995); + + let saved = to_reduced_previous_quorums(&source).expect("should capture the history"); + + let mut restored = SignatureVerificationQuorumSet::new( + &ChainLockConfig::default_100_67(), + PlatformVersion::latest(), + ) + .expect("should build quorum set"); + restore_previous_quorums(&mut restored, Some(&saved)).expect("should restore"); + + let source_previous = source.previous_past_quorums().expect("source has history"); + let restored_previous = restored + .previous_past_quorums() + .expect("restored must have history"); + + assert_eq!( + restored_previous.last_active_core_height, + source_previous.last_active_core_height + ); + assert_eq!( + restored_previous.updated_at_core_height, + source_previous.updated_at_core_height + ); + assert_eq!( + restored_previous.previous_change_height, + source_previous.previous_change_height + ); + assert_eq!(restored_previous.previous_change_height, Some(950)); + + assert_eq!( + restored_previous.quorums.len(), + source_previous.quorums.len() + ); + for (quorum_hash, source_quorum) in source_previous.quorums.iter() { + let restored_quorum = restored_previous + .quorums + .get(quorum_hash) + .expect("every quorum must be restored"); + assert_eq!(restored_quorum.public_key, source_quorum.public_key); + assert_eq!(restored_quorum.index, source_quorum.index); + } + } + + /// A set with no history restores to no history, not to an empty one — an empty + /// previous set would make `select_quorums` consider locks verifiable against nothing. + #[test] + fn should_leave_a_set_without_history_alone() { + use crate::config::ChainLockConfig; + + let mut restored = SignatureVerificationQuorumSet::new( + &ChainLockConfig::default_100_67(), + PlatformVersion::latest(), + ) + .expect("should build quorum set"); + restore_previous_quorums(&mut restored, None).expect("should restore"); + assert!(!restored.has_previous_past_quorums()); + } + + #[test] + fn should_sort_validator_sets_into_saved_positions() { + use dpp::bls_signatures::{Bls12381G2Impl, SecretKey}; + use dpp::core_types::validator_set::v0::ValidatorSetV0; + use rand::rngs::StdRng; + use rand::SeedableRng; + + let mut rng = StdRng::seed_from_u64(7); + let mut validator_sets: IndexMap = IndexMap::new(); + for seed in [1u8, 2, 3] { + validator_sets.insert( + quorum_hash(seed), + ValidatorSet::V0(ValidatorSetV0 { + quorum_hash: quorum_hash(seed), + quorum_index: None, + core_height: 100, + members: Default::default(), + threshold_public_key: SecretKey::::random(&mut rng) + .public_key(), + }), + ); + } + + let saved_positions: Vec = [3u8, 1, 2] + .into_iter() + .map(|seed| quorum_hash(seed).to_byte_array().into()) + .collect(); + + sort_validator_sets_by_saved_positions(&mut validator_sets, &saved_positions); + + let order: Vec = validator_sets.keys().copied().collect(); + assert_eq!(order, vec![quorum_hash(3), quorum_hash(1), quorum_hash(2)]); + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs new file mode 100644 index 00000000000..c7d52ae3e10 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs @@ -0,0 +1,34 @@ +mod v0; + +use crate::error::execution::ExecutionError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + /// Fetch the reduced platform state from the replicated grovedb state. + /// + /// Returns `Ok(None)` when the reduced state is absent (a snapshot taken before the + /// protocol version that introduced it). + pub fn fetch_reduced_platform_state( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + match platform_version + .drive_abci + .methods + .platform_state_storage + .fetch_reduced_platform_state + { + 0 => self.fetch_reduced_platform_state_v0(transaction, platform_version), + version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { + method: "fetch_reduced_platform_state".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs new file mode 100644 index 00000000000..253c299e055 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs @@ -0,0 +1,22 @@ +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::serialization::PlatformDeserializable; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + pub(super) fn fetch_reduced_platform_state_v0( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + self.drive + .fetch_reduced_platform_state_bytes(transaction, platform_version) + .map_err(Error::Drive)? + .map(|bytes| { + ReducedPlatformState::deserialize_from_bytes(&bytes).map_err(Error::Protocol) + }) + .transpose() + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/mod.rs b/packages/rs-drive-abci/src/execution/storage/mod.rs index 92c2b2417dc..017babf8c28 100644 --- a/packages/rs-drive-abci/src/execution/storage/mod.rs +++ b/packages/rs-drive-abci/src/execution/storage/mod.rs @@ -1,2 +1,4 @@ pub mod fetch_platform_state; +mod fetch_reduced_platform_state; mod store_platform_state; +mod store_reduced_platform_state; diff --git a/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs new file mode 100644 index 00000000000..5b037ca8d39 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs @@ -0,0 +1,32 @@ +mod v0; + +use crate::error::execution::ExecutionError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + /// Store the reduced platform state in the replicated grovedb state + pub fn store_reduced_platform_state( + &self, + state: &ReducedPlatformState, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match platform_version + .drive_abci + .methods + .platform_state_storage + .store_reduced_platform_state + { + 0 => self.store_reduced_platform_state_v0(state, transaction, platform_version), + version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { + method: "store_reduced_platform_state".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs new file mode 100644 index 00000000000..aa4db11a82e --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs @@ -0,0 +1,23 @@ +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::serialization::PlatformSerializable; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + pub(super) fn store_reduced_platform_state_v0( + &self, + state: &ReducedPlatformState, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + self.drive + .store_reduced_platform_state_bytes( + &state.serialize_to_bytes()?, + transaction, + platform_version, + ) + .map_err(Error::Drive) + } +} diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/masternode_vote/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/masternode_vote/mod.rs index 3707d37c101..619e3b5120b 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/masternode_vote/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/masternode_vote/mod.rs @@ -94,6 +94,7 @@ impl StateTransitionStateValidation for MasternodeVoteTransition { #[cfg(test)] mod tests { + use crate::test::helpers::fast_forward_to_block::TEST_BLOCK_SIGNATURE; use crate::test::helpers::setup::TestPlatformBuilder; use dpp::block::block_info::BlockInfo; use dpp::dash_to_credits; @@ -3065,7 +3066,7 @@ mod tests { offset: None, limit: None, start_at: None, - order_ascending: true, + order_ascending, }; let (_, voters) = resolved_contested_document_vote_poll_drive_query @@ -4214,7 +4215,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4406,7 +4407,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4561,7 +4562,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4604,7 +4605,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4823,7 +4824,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4866,7 +4867,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4987,7 +4988,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5030,7 +5031,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5148,7 +5149,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5191,7 +5192,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5325,7 +5326,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5995,7 +5996,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -6218,7 +6219,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -6505,7 +6506,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -6951,7 +6952,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -7074,7 +7075,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -7434,7 +7435,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -7652,7 +7653,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -7865,7 +7866,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8067,7 +8068,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8285,7 +8286,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8499,7 +8500,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8694,7 +8695,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8889,7 +8890,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9193,7 +9194,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9383,7 +9384,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9448,7 +9449,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9718,7 +9719,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9777,7 +9778,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9971,7 +9972,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10289,7 +10290,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10481,7 +10482,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10558,7 +10559,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10757,7 +10758,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10984,7 +10985,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -11502,7 +11503,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), diff --git a/packages/rs-drive-abci/src/platform_types/mod.rs b/packages/rs-drive-abci/src/platform_types/mod.rs index 0f3b33981de..7f59de23585 100644 --- a/packages/rs-drive-abci/src/platform_types/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/mod.rs @@ -22,6 +22,8 @@ pub mod platform_state; pub mod required_identity_public_key_set; /// Signature verification quorums for Core pub mod signature_verification_quorum_set; +/// ABCI state sync snapshot types +pub mod snapshot; /// The state transition execution result as part of the block execution outcome pub mod state_transitions_processing_result; /// The validator module diff --git a/packages/rs-drive-abci/src/platform_types/platform/mod.rs b/packages/rs-drive-abci/src/platform_types/platform/mod.rs index 972e530a1ce..514fbabce09 100644 --- a/packages/rs-drive-abci/src/platform_types/platform/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform/mod.rs @@ -10,6 +10,9 @@ use std::fmt::{Debug, Formatter}; use crate::platform_types::check_tx_proof_verifier::CheckTxProofVerifier; use crate::platform_types::platform_state::{PlatformState, PlatformStateV0Methods}; +use crate::platform_types::snapshot::{ + clear_restore_sentinel, restore_sentinel_exists, wipe_drive_for_restore, +}; use arc_swap::ArcSwap; use dpp::prelude::BlockHeight; use dpp::serialization::PlatformDeserializableFromVersionedStructure; @@ -144,8 +147,54 @@ impl Platform { } }; - let (drive, current_platform_version) = - Drive::open(&config.db_path, Some(config.drive.clone())).map_err(Error::Drive)?; + // Checkpoints are created under the operator-configured `CHECKPOINTS_PATH` + // (defaulting to `/checkpoints`); startup MUST read them back from the + // same place, or a node with a custom path comes up with an empty registry: + // it would stop advertising the snapshots it retained and could never prune the + // directories it wrote. + let checkpoints_path = config + .abci + .state_sync + .resolved_checkpoints_path(&config.db_path); + + let (drive, current_platform_version) = Drive::open_with_checkpoints_path( + &config.db_path, + Some(config.drive.clone()), + &checkpoints_path, + ) + .map_err(Error::Drive)?; + + // A state sync restore that never finished leaves grovedb holding state the + // platform state knows nothing about. That is not recoverable by restarting — + // the `info` handler panics on the mismatch, so the node would crash-loop — and it + // cannot be told apart from corruption without a marker. `offer_snapshot` writes + // one before it wipes; if it is still here, the restore did not finish. + // + // Recovery is to become an empty node: wipe, drop the caches derived from what was + // wiped, and come up as if freshly installed, so Tenderdash can offer another + // snapshot or fall back to block sync. Clearing the marker afterwards is safe + // precisely because an empty database with no saved state is self-consistent. + let current_platform_version = if restore_sentinel_exists(&config.db_path) { + tracing::warn!( + db_path = ?config.db_path, + "[state_sync] an unfinished state sync restore was found on startup; wiping \ + and coming up empty so the node can sync again", + ); + + wipe_drive_for_restore(&drive).map_err(Error::Drive)?; + clear_restore_sentinel(&config.db_path).map_err(|e| { + Error::Drive(drive::error::Error::IOErrorWithInfoString( + e.into(), + "trying to clear the state sync restore sentinel".to_owned(), + )) + })?; + + // The wipe removed the stored protocol version along with everything else, so + // the saved-state branch below must not be taken. + None + } else { + current_platform_version + }; if let Some(platform_version) = current_platform_version { let Some(execution_state) = @@ -160,9 +209,7 @@ impl Platform { let mut checkpoint_platform_states = BTreeMap::new(); let checkpoints = drive.checkpoints.load(); for (&block_height, _checkpoint_info) in checkpoints.iter() { - let checkpoint_state_path = config - .db_path - .join("checkpoints") + let checkpoint_state_path = checkpoints_path .join(block_height.to_string()) .join("platform_state.bin"); diff --git a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs index 81f438fe51a..ca4edfd610f 100644 --- a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs @@ -21,11 +21,18 @@ use crate::error::execution::ExecutionError; pub use crate::platform_types::platform_state::accessors::PlatformStateV0Methods; use crate::platform_types::platform_state::platform_state_for_saving::v1::PlatformStateForSavingV1; use crate::platform_types::platform_state::platform_state_for_saving::PlatformStateForSaving; -use crate::platform_types::signature_verification_quorum_set::SignatureVerificationQuorumSet; +use crate::platform_types::signature_verification_quorum_set::{ + SignatureVerificationQuorumSet, SignatureVerificationQuorumSetV0Methods, +}; use dpp::block::block_info::BlockInfo; use dpp::dashcore::hashes::Hash; use dpp::dashcore_rpc::json::MasternodeListItem; use dpp::fee::default_costs::CachedEpochIndexFeeVersions; +use dpp::reduced_platform_state::v0::{ + ReducedBlockInfoV0, ReducedPlatformStateV0, ReducedPreviousQuorumsV0, + ReducedVerificationQuorumV0, +}; +use dpp::reduced_platform_state::ReducedPlatformState; use dpp::util::hash::hash_double; use std::collections::BTreeMap; use std::fmt::{Debug, Formatter}; @@ -122,6 +129,50 @@ impl PlatformState { pub fn fingerprint(&self) -> Result<[u8; 32], Error> { Ok(hash_double(self.serialize_to_bytes()?)) } + + /// Builds the reduced platform state that is written into the replicated grovedb + /// state each block so a state-synced node can reconstruct the full platform state. + /// + /// `last_committed_block_info` describes the block currently being processed (it + /// becomes the last committed block once the block finalizes); fields that are not + /// known during proposal processing (app hash, block id hash, signature) are `None`. + /// `quorum_positions` records the order of the validator sets, which is not + /// otherwise recoverable from Core RPC during reconstruction. + pub fn to_reduced_platform_state( + &self, + last_committed_block_info: Option, + proposed_core_chain_locked_height: u32, + ) -> ReducedPlatformState { + ReducedPlatformState::V0(ReducedPlatformStateV0 { + last_committed_block_info, + current_protocol_version_in_consensus: self.current_protocol_version_in_consensus, + next_epoch_protocol_version: self.next_epoch_protocol_version, + current_validator_set_quorum_hash: self + .current_validator_set_quorum_hash + .to_byte_array() + .into(), + next_validator_set_quorum_hash: self + .next_validator_set_quorum_hash + .map(|quorum_hash| quorum_hash.to_byte_array().into()), + previous_fee_versions: self + .previous_fee_versions + .iter() + .map(|(epoch_index, fee_version)| (*epoch_index, fee_version.fee_version_number)) + .collect(), + quorum_positions: self + .validator_sets + .keys() + .map(|quorum_hash| quorum_hash.to_byte_array().into()) + .collect(), + proposed_core_chain_locked_height, + previous_chain_lock_quorums: to_reduced_previous_quorums( + &self.chain_lock_validating_quorums, + ), + previous_instant_lock_quorums: to_reduced_previous_quorums( + &self.instant_lock_validating_quorums, + ), + }) + } /// The default state at init chain pub fn default_with_protocol_versions( current_protocol_version_in_consensus: ProtocolVersion, @@ -155,6 +206,33 @@ impl PlatformState { } } +/// Captures the superseded quorums of a signature-verification quorum set for the reduced +/// platform state. +/// +/// The CURRENT quorums are deliberately not captured: reconstruction re-derives them from +/// Core, where the set at a given core height is exactly what Core reports. The history is +/// the part Core cannot answer, so it is the part that has to be carried. +pub(crate) fn to_reduced_previous_quorums( + quorum_set: &SignatureVerificationQuorumSet, +) -> Option { + let previous = quorum_set.previous_past_quorums()?; + + Some(ReducedPreviousQuorumsV0 { + quorums: previous + .quorums + .iter() + .map(|(quorum_hash, quorum)| ReducedVerificationQuorumV0 { + quorum_hash: quorum_hash.to_byte_array().into(), + public_key: quorum.public_key.0.to_compressed(), + index: quorum.index, + }) + .collect(), + last_active_core_height: previous.last_active_core_height, + updated_at_core_height: previous.updated_at_core_height, + previous_change_height: previous.previous_change_height, + }) +} + impl PlatformSerializable for PlatformState { type Error = Error; diff --git a/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/mod.rs b/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/mod.rs index 1d4a9e71967..df36b83b9f6 100644 --- a/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/mod.rs @@ -7,8 +7,8 @@ use crate::platform_types::signature_verification_quorum_set::v0::for_saving_v0: use crate::platform_types::signature_verification_quorum_set::v0::for_saving_v1::SignatureVerificationQuorumSetForSavingV1; use crate::platform_types::signature_verification_quorum_set::v0::for_saving_v2::SignatureVerificationQuorumSetForSavingV2; pub use crate::platform_types::signature_verification_quorum_set::v0::quorum_set::{ - QuorumConfig, QuorumsWithConfig, SelectedQuorumSetIterator, SignatureVerificationQuorumSetV0, - SignatureVerificationQuorumSetV0Methods, SIGN_OFFSET, + PreviousPastQuorums, QuorumConfig, QuorumsWithConfig, SelectedQuorumSetIterator, + SignatureVerificationQuorumSetV0, SignatureVerificationQuorumSetV0Methods, SIGN_OFFSET, }; pub use crate::platform_types::signature_verification_quorum_set::v0::quorums::{ Quorum, Quorums, SigningQuorum, ThresholdBlsPublicKey, VerificationQuorum, @@ -76,6 +76,29 @@ impl SignatureVerificationQuorumSetV0Methods for SignatureVerificationQuorumSet } } + fn previous_past_quorums(&self) -> Option> { + match self { + Self::V0(v0) => v0.previous_past_quorums(), + } + } + + fn restore_previous_past_quorums( + &mut self, + previous_quorums: Quorums, + last_active_core_height: u32, + updated_at_core_height: u32, + previous_change_height: Option, + ) { + match self { + Self::V0(v0) => v0.restore_previous_past_quorums( + previous_quorums, + last_active_core_height, + updated_at_core_height, + previous_change_height, + ), + } + } + fn replace_quorums( &mut self, quorums: Quorums, diff --git a/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/v0/quorum_set.rs b/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/v0/quorum_set.rs index 881347bc6ae..854f47087a0 100644 --- a/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/v0/quorum_set.rs +++ b/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/v0/quorum_set.rs @@ -23,6 +23,18 @@ pub(super) struct PreviousPastQuorumsV0 { pub(super) previous_change_height: Option, } +/// A borrowed view of the superseded quorums of a set, for callers outside this module. +pub struct PreviousPastQuorums<'q> { + /// The superseded quorums + pub quorums: &'q Quorums, + /// The core height at which these quorums were last active + pub last_active_core_height: u32, + /// The core height at which the quorums were changed + pub updated_at_core_height: u32, + /// The core height at which the set before these became active + pub previous_change_height: Option, +} + /// Quorums with keys for signature verification #[derive(Debug, Clone)] pub struct SignatureVerificationQuorumSetV0 { @@ -55,6 +67,27 @@ pub trait SignatureVerificationQuorumSetV0Methods { /// Has previous quorums? fn has_previous_past_quorums(&self) -> bool; + /// The superseded quorums and the core heights that bound their validity, if any. + /// + /// This history exists only in the platform state — it cannot be re-derived from Core + /// — so it has to be readable to travel with a state sync snapshot. + fn previous_past_quorums(&self) -> Option>; + + /// Restores the superseded quorums verbatim, including the change height of the set + /// before them. + /// + /// Unlike [`SignatureVerificationQuorumSetV0Methods::set_previous_past_quorums`], this + /// does NOT derive `previous_change_height` from whatever this set currently holds: it + /// is for reinstating a history that was captured elsewhere (state sync reconstruction), + /// where deriving would silently produce a different one. + fn restore_previous_past_quorums( + &mut self, + previous_quorums: Quorums, + last_active_core_height: u32, + updated_at_core_height: u32, + previous_change_height: Option, + ); + /// Set last quorums keys and update previous quorums fn replace_quorums( &mut self, @@ -172,6 +205,30 @@ impl SignatureVerificationQuorumSetV0Methods for SignatureVerificationQuorumSetV self.previous.is_some() } + fn previous_past_quorums(&self) -> Option> { + self.previous.as_ref().map(|previous| PreviousPastQuorums { + quorums: &previous.quorums, + last_active_core_height: previous.last_active_core_height, + updated_at_core_height: previous.updated_at_core_height, + previous_change_height: previous.previous_change_height, + }) + } + + fn restore_previous_past_quorums( + &mut self, + previous_quorums: Quorums, + last_active_core_height: u32, + updated_at_core_height: u32, + previous_change_height: Option, + ) { + self.previous = Some(PreviousPastQuorumsV0 { + quorums: previous_quorums, + last_active_core_height, + updated_at_core_height, + previous_change_height, + }); + } + fn replace_quorums( &mut self, quorums: Quorums, diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs new file mode 100644 index 00000000000..43b2d7b8862 --- /dev/null +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -0,0 +1,543 @@ +//! ABCI state sync snapshot types. +//! +//! Snapshots are served directly from the rocksdb checkpoints Drive already creates +//! (`drive.checkpoints`, populated by `create_grovedb_checkpoint` after each qualifying +//! block is committed); there is no separate snapshot store. + +use dpp::util::deserializer::ProtocolVersion; +use dpp::version::PlatformVersion; +use drive::drive::{Checkpoint, Drive}; +use drive::grovedb::replication::MultiStateSyncSession; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; +use tenderdash_abci::proto::abci; + +/// Name of the marker file that records "a state sync restore is in progress". +/// +/// ## Why a plain file, and not aux storage +/// +/// The obvious home would be grovedb's aux column family, which is not part of the +/// provable tree. It cannot be used here: `GroveDb::wipe()` clears the aux column family +/// along with `default`, `roots` and `meta` +/// (`grovedb/storage/src/rocksdb_storage/storage.rs`, `wipe()` iterates all four). Since +/// wiping is exactly what both the offer path and the recovery path do, a sentinel living +/// in aux would be destroyed by the very operations it exists to survive, and its +/// lifetime would depend on subtle ordering between the write and the wipe. +/// +/// A file next to the database has none of those problems: it is outside everything +/// grovedb touches, it survives any wipe, it costs one `stat` at startup, and an operator +/// can see it. It is deliberately NOT in the provable tree either — it is node-local +/// recovery bookkeeping and must never affect the app hash. +pub const RESTORE_IN_PROGRESS_FILE_NAME: &str = "state_sync_restore_in_progress"; + +/// Path of the restore sentinel for a given database directory. +pub fn restore_sentinel_path(db_path: &Path) -> PathBuf { + db_path.join(RESTORE_IN_PROGRESS_FILE_NAME) +} + +/// Records that a state sync restore has started and the database is therefore allowed to +/// be inconsistent until it finishes. +/// +/// Written BEFORE the wipe, so the window in which the database has been destroyed but +/// nothing marks it as such is empty. The contents are for operators only; the code cares +/// solely about the file's presence. +pub fn write_restore_sentinel( + db_path: &Path, + app_hash: &[u8; 32], + height: u64, +) -> std::io::Result<()> { + std::fs::create_dir_all(db_path)?; + std::fs::write( + restore_sentinel_path(db_path), + format!( + "state sync restore in progress\nheight: {}\napp_hash: {}\n", + height, + hex::encode(app_hash) + ), + ) +} + +/// Clears the restore sentinel at a point where the node is already self-consistent — +/// after a completed restore, or after a genesis initialization — WITHOUT being able to +/// fail the operation that got it there. +/// +/// Propagating an I/O error from here would turn a fully successful restore (or a working +/// genesis) into a hard ABCI error over nothing but a `remove_file` hiccup. The cost of +/// failing to remove it is bounded and safe in the other direction: the next startup sees +/// a sentinel, wipes, and re-syncs. Loud, but never a wedge. +pub fn clear_restore_sentinel_best_effort(db_path: &Path) { + if let Err(error) = clear_restore_sentinel(db_path) { + tracing::error!( + ?error, + path = ?restore_sentinel_path(db_path), + "[state_sync] could not clear the state sync restore sentinel; the node is \ + consistent, but the next restart will wipe and re-sync unnecessarily. Remove \ + the file by hand to avoid that.", + ); + } +} + +/// Clears the restore sentinel. Only ever called once the node is in a self-consistent +/// state: after a restore has fully completed, after startup recovery has wiped, or after +/// a genesis initialization. +pub fn clear_restore_sentinel(db_path: &Path) -> std::io::Result<()> { + match std::fs::remove_file(restore_sentinel_path(db_path)) { + Ok(()) => Ok(()), + // Absent is the normal case on every path that clears defensively. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +/// Whether a restore was in progress when this node last stopped. +pub fn restore_sentinel_exists(db_path: &Path) -> bool { + restore_sentinel_path(db_path).exists() +} + +/// Drops every Drive cache that was derived from grovedb. +/// +/// A wipe destroys the state these caches were built from. Left in place they would be +/// silently merged into whatever replaces it: `ProtocolVersionsCache` in particular keeps +/// a `loaded` flag, so `load_if_needed` would never re-read the new version counters and +/// the next block would write vote counts derived from the wiped chain — an immediate app +/// hash fork. Resetting the counter wholesale (rather than `clear_global_cache`) is +/// deliberate: it clears that flag too, so the cache reloads on first use. +/// +/// `system_data_contracts` is deliberately NOT cleared — those are compiled-in, +/// version-keyed contracts that never come from grovedb. +/// +/// The checkpoint registry goes too, and it is not merely a cache: `Drive::open` populates +/// `drive.checkpoints` before any wipe can run, and `list_snapshots` serves whatever is in +/// it to peers. Left alone, a node that wiped and re-synced would keep offering snapshots +/// of the chain it just discarded. The entries are marked for deletion first so their +/// directories are removed when the last `Arc` drops, rather than leaking on disk. +pub fn reset_drive_caches_after_wipe(drive: &Drive) { + *drive.cache.protocol_versions_counter.write() = Default::default(); + drive.cache.data_contracts.clear(); + *drive.cache.genesis_time_ms.write() = None; + + let checkpoints = drive.checkpoints.load(); + for checkpoint_info in checkpoints.values() { + checkpoint_info.checkpoint.mark_for_deletion(); + } + drive.checkpoints.store(Arc::new(BTreeMap::new())); +} + +/// Wipes grovedb and drops the caches derived from it, leaving the node an empty but +/// entirely self-consistent slate. +/// +/// This is the single place both the offer path and the crash-recovery path go through, +/// so the two can never drift apart. +pub fn wipe_drive_for_restore(drive: &Drive) -> Result<(), drive::error::Error> { + drive.grove.wipe()?; + reset_drive_caches_after_wipe(drive); + Ok(()) +} + +/// The grovedb state sync protocol versions this node can serve and consume. +/// +/// Exactly one protocol version exists: state sync never shipped, so grovedb updates +/// its replication protocol in place and stays at version 1. This single supported-set +/// constant and the offered-snapshot validation against it exist so that any future +/// incompatible protocol change fails fast on both the serving and consuming side +/// instead of producing a corrupt restore. (`drive_abci.state_sync.protocol_version` +/// is the version stamped on snapshots this node offers.) +pub const SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS: &[u16] = &[1]; + +/// Maximum accepted size (in bytes) of a single snapshot chunk, enforced before any +/// grovedb decode of peer-supplied data (issue #3773). +pub const MAX_STATE_SYNC_CHUNK_SIZE: usize = 16 * 1024 * 1024; + +/// Maximum accepted size (in bytes) of a chunk id, enforced before any grovedb decode +/// of peer-supplied data (issue #3773). Chunk ids are packed vectors of 32-byte subtree +/// prefixes plus short traversal instructions, so well-formed ids stay far below this. +pub const MAX_STATE_SYNC_CHUNK_ID_SIZE: usize = 64 * 1024; + +/// Maximum number of subtrees processed in a single batch of a grovedb state sync +/// session on the consuming side. +pub const STATE_SYNC_SUBTREES_BATCH_SIZE: usize = 64; + +/// Encodes the Platform protocol version a snapshot was produced at into the ABCI +/// snapshot `metadata` field. +/// +/// The consuming node cannot derive this: a node that state syncs has no saved state, so +/// its in-memory platform state is still at [`dpp::version::INITIAL_PROTOCOL_VERSION`] +/// and its Drive version table — including `grove_version` — is the wrong one for the +/// snapshot. grovedb's replication, tree opening and root-hash rules are version gated, +/// so serving and consuming MUST use the same table or the restore is decoded under +/// different rules than it was generated with. +/// +/// The value is peer-supplied and therefore untrusted, which is safe: the restore is only +/// ever accepted against the light-client-verified app hash, so a lie produces a failed +/// verification and a `REJECT_SNAPSHOT`, never a silently wrong database. +pub fn encode_snapshot_metadata(protocol_version: ProtocolVersion) -> Vec { + protocol_version.to_be_bytes().to_vec() +} + +/// Decodes the Platform protocol version out of an ABCI snapshot's `metadata` field. +/// Returns `None` for anything that is not exactly the encoding above. +pub fn decode_snapshot_metadata(metadata: &[u8]) -> Option { + <[u8; 4]>::try_from(metadata) + .ok() + .map(ProtocolVersion::from_be_bytes) +} + +/// A state sync transfer in progress on the consuming side. +pub struct SnapshotFetchingSession<'db> { + /// The snapshot being restored + pub snapshot: abci::Snapshot, + /// The light-client-verified app hash for the snapshot height, from Tenderdash + pub app_hash: [u8; 32], + /// The grovedb state sync wire protocol version this transfer speaks — taken from + /// the offered snapshot's `version`, validated against + /// [`SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS`], and used for every chunk of the + /// transfer. + pub wire_version: u16, + /// The Platform version the snapshot was PRODUCED at, decoded from the offered + /// snapshot's metadata (see [`encode_snapshot_metadata`]). Every grovedb call of this + /// transfer — session start, chunk application, commit, verification and the final + /// root hash — uses this version's table, not the fresh node's own. + pub platform_version: &'static PlatformVersion, + /// The grovedb state sync session + pub state_sync_info: Pin>>, +} + +/// How long a served checkpoint stays pinned after the last chunk request for it. +/// +/// A state-syncing peer requests chunks continuously; if none arrived for this long the +/// transfer is considered abandoned and the pin is released, allowing a checkpoint that +/// pruning already marked for deletion to be removed from disk. +const SERVING_PIN_INACTIVITY_TTL: Duration = Duration::from_secs(600); + +/// Absolute lifetime of a serving pin, regardless of activity. +/// +/// The inactivity TTL alone is refreshable, so a peer that keeps touching a height keeps +/// its checkpoint alive forever; this deadline is deliberately NOT refreshable. +/// +/// It is the backstop, not the primary bound — [`max_serving_pins`] is what actually +/// limits how many directories can be held back at once — so it is set well above any +/// plausible honest transfer rather than tight. A full mainnet-state restore measures in +/// seconds; six hours leaves enormous room for a slow or rate-limited peer whose +/// checkpoint gets pruned mid-transfer, while still bounding how long a pinned directory +/// can outlive its checkpoint. +const SERVING_PIN_MAX_LIFETIME: Duration = Duration::from_secs(6 * 3600); + +/// How often the autonomous sweep task releases expired serving pins. +/// +/// Expiry must not depend on peers making further requests (an abandoned transfer makes +/// none) nor on this process seeing blocks (the gRPC serving application does not). +/// The interval only bounds how long an expired pin lingers past its deadline, so it is +/// uncritical; once a minute is nothing next to the TTLs it enforces. +pub const SERVING_PIN_SWEEP_INTERVAL: Duration = Duration::from_secs(60); + +/// How many pins are allowed on top of the number of snapshots the node retains. +/// +/// The interesting pins are the ones for checkpoints pruning has ALREADY dropped from the +/// registry — those are the directories a pin holds back from deletion. There can only +/// ever be a handful of them legitimately (a transfer that started before the checkpoint +/// aged out), so the retained count plus this slack is generous. +const SERVING_PIN_SLACK: usize = 4; + +/// Cap on how many checkpoints may be pinned for serving at once, given how many snapshots +/// the node is configured to retain. +/// +/// Without a cap, a peer could keep one transfer alive per height it ever touched: pruning +/// keeps advancing, the peer keeps refreshing, and the number of checkpoint directories +/// held back from deletion grows without bound regardless of `MAX_NUM_SNAPSHOTS`. The cap +/// only ever bites on abuse; when it does, the least recently served pin goes first, and a +/// peer whose pin is evicted can still resolve the checkpoint from the registry if it is +/// still there. +pub fn max_serving_pins(max_num_snapshots: usize) -> usize { + max_num_snapshots.saturating_add(SERVING_PIN_SLACK) +} + +/// A checkpoint held back from deletion for a transfer in flight. +struct ServingPin { + checkpoint: Arc, + /// When the pin was first taken — bounds its absolute lifetime + pinned_at: Instant, + /// When a chunk was last successfully served from it — bounds its idle lifetime + last_served: Instant, +} + +impl ServingPin { + fn is_live(&self, now: Instant) -> bool { + now.saturating_duration_since(self.last_served) < SERVING_PIN_INACTIVITY_TTL + && now.saturating_duration_since(self.pinned_at) < SERVING_PIN_MAX_LIFETIME + } +} + +/// Keeps checkpoints that are actively being served to state-syncing peers alive. +/// +/// Checkpoint pruning marks old checkpoints for deletion and drops them from the +/// registry; the directory is removed when the last `Arc` drops. Holding an +/// `Arc` clone here for every checkpoint a peer is currently downloading extends that +/// refcount, so a checkpoint cannot be deleted mid-transfer. +/// +/// A pin must not be something a remote peer can hold open indefinitely, so it is bounded +/// three ways: [`SERVING_PIN_INACTIVITY_TTL`] since the last chunk actually served, +/// [`SERVING_PIN_MAX_LIFETIME`] since it was taken (not refreshable), and +/// [`max_serving_pins`] in total. Expiry also must not depend on peers making further +/// requests, or an abandoned transfer would hold its directory forever: +/// [`SnapshotManager::release_expired_pins`] runs autonomously (every +/// [`SERVING_PIN_SWEEP_INTERVAL`] from the sweep task `server::start` spawns next to the +/// gRPC serving application, and once per block in the all-in-one test application) and +/// every read of a pin re-checks both deadlines. +#[derive(Default)] +pub struct SnapshotManager { + /// Height -> the pin held for a transfer of that snapshot + serving_pins: RwLock>, +} + +impl SnapshotManager { + /// Creates a new snapshot manager with no active pins + pub fn new() -> Self { + Self::default() + } + + /// Pins a checkpoint that is being served (or refreshes the pin of one that already + /// is), dropping expired pins and holding the total to `max_pins` (see + /// [`max_serving_pins`]). + /// + /// Call this only AFTER a chunk was successfully served: a request that could not be + /// answered must not be able to keep a checkpoint alive. + pub fn pin_for_serving(&self, height: u64, checkpoint: Arc, max_pins: usize) { + let now = Instant::now(); + let mut pins = self + .serving_pins + .write() + .expect("serving pins lock poisoned"); + retain_live_pins(&mut pins, now); + + if let Some(pin) = pins.get_mut(&height) { + // Refresh the idle deadline only — `pinned_at` is deliberately untouched so + // the absolute lifetime cannot be extended by activity. + pin.last_served = now; + return; + } + + // Evict the least recently served pin to make room for a genuinely new one + while pins.len() >= max_pins.max(1) { + let Some(coldest) = pins + .iter() + .min_by_key(|(_, pin)| pin.last_served) + .map(|(pinned_height, _)| *pinned_height) + else { + break; + }; + tracing::warn!( + evicted_height = coldest, + new_height = height, + "[state_sync] serving pin limit reached, releasing the least recently served pin", + ); + pins.remove(&coldest); + } + + pins.insert( + height, + ServingPin { + checkpoint, + pinned_at: now, + last_served: now, + }, + ); + } + + /// Returns a pinned checkpoint for the given height, if the pin is still held AND + /// still live. + /// + /// Used to keep serving a snapshot whose checkpoint pruning has already dropped from + /// the registry. An expired pin is dropped rather than returned: handing one out + /// would let a peer resurrect (and then indefinitely refresh) a checkpoint whose + /// transfer was abandoned long ago. + pub fn pinned_checkpoint(&self, height: u64) -> Option> { + let now = Instant::now(); + let mut pins = self + .serving_pins + .write() + .expect("serving pins lock poisoned"); + retain_live_pins(&mut pins, now); + pins.get(&height).map(|pin| Arc::clone(&pin.checkpoint)) + } + + /// Releases every pin that has passed either of its deadlines. + /// + /// Called once per block so an abandoned transfer cannot hold a pruned checkpoint + /// directory on disk forever while waiting for a chunk request that never comes, and + /// so an over-long one is cut off even while it keeps requesting. + pub fn release_expired_pins(&self) { + let now = Instant::now(); + let mut pins = self + .serving_pins + .write() + .expect("serving pins lock poisoned"); + retain_live_pins(&mut pins, now); + } + + /// Number of checkpoints currently pinned for serving. + #[cfg(test)] + pub fn pinned_count(&self) -> usize { + self.serving_pins + .read() + .expect("serving pins lock poisoned") + .len() + } + + /// Test-only: the two deadlines of a pin, as `(pinned_at, last_served)`. + #[cfg(test)] + fn pin_instants(&self, height: u64) -> Option<(Instant, Instant)> { + self.serving_pins + .read() + .expect("serving pins lock poisoned") + .get(&height) + .map(|pin| (pin.pinned_at, pin.last_served)) + } + + /// Test-only: backdates a pin's deadlines so expiry can be exercised without waiting. + #[cfg(test)] + fn backdate_pin(&self, height: u64, pinned_at: Option, last_served: Option) { + let mut pins = self + .serving_pins + .write() + .expect("serving pins lock poisoned"); + if let Some(pin) = pins.get_mut(&height) { + if let Some(pinned_at) = pinned_at { + pin.pinned_at = pinned_at; + } + if let Some(last_served) = last_served { + pin.last_served = last_served; + } + } + } +} + +fn retain_live_pins(pins: &mut BTreeMap, now: Instant) { + pins.retain(|_, pin| pin.is_live(now)); +} + +#[cfg(test)] +mod tests { + use super::*; + use drive::grovedb::GroveDb; + + /// Each checkpoint needs its own directory: rocksdb takes an exclusive lock on the + /// one it opens. + fn checkpoint_in(dir: &tempfile::TempDir, height: u64) -> Arc { + let path = dir.path().join(height.to_string()); + let grove_db = GroveDb::open(&path).expect("should open grovedb"); + Arc::new(Checkpoint::new(grove_db, path)) + } + + /// An abandoned transfer stops making chunk requests, so the pin must expire on its + /// own — both when swept from the block path and when a later request tries to read + /// it (otherwise a peer could resurrect and then indefinitely refresh a checkpoint + /// pruning already dropped). + #[test] + fn expired_pins_are_released_without_any_further_chunk_request() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let manager = SnapshotManager::new(); + manager.pin_for_serving(10, checkpoint_in(&dir, 10), max_serving_pins(3)); + assert!(manager.pinned_checkpoint(10).is_some()); + + let Some(long_ago) = Instant::now().checked_sub(SERVING_PIN_INACTIVITY_TTL * 2) else { + // Monotonic clock too young to backdate; nothing to assert on this platform. + return; + }; + manager.backdate_pin(10, None, Some(long_ago)); + + assert!( + manager.pinned_checkpoint(10).is_none(), + "an expired pin must not be handed out", + ); + + manager.pin_for_serving(11, checkpoint_in(&dir, 11), max_serving_pins(3)); + manager.backdate_pin(11, None, Some(long_ago)); + manager.release_expired_pins(); + assert_eq!( + manager.pinned_count(), + 0, + "the block-driven sweep must release expired pins with no peer activity", + ); + } + + /// The inactivity TTL is refreshable, so on its own it lets a peer hold a checkpoint + /// forever by touching it periodically. The absolute lifetime is not refreshable and + /// must cut such a pin off even while requests keep arriving. + #[test] + fn constant_touching_cannot_extend_a_pin_past_its_absolute_lifetime() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let manager = SnapshotManager::new(); + let checkpoint = checkpoint_in(&dir, 10); + manager.pin_for_serving(10, Arc::clone(&checkpoint), max_serving_pins(3)); + let (pinned_at, _) = manager.pin_instants(10).expect("pin must exist"); + + // Continued activity refreshes the idle deadline but must NOT reset `pinned_at`, + // or the absolute deadline could be pushed out forever. + manager.pin_for_serving(10, Arc::clone(&checkpoint), max_serving_pins(3)); + let (pinned_at_after_touch, last_served) = + manager.pin_instants(10).expect("pin must exist"); + assert_eq!( + pinned_at, pinned_at_after_touch, + "serving another chunk must not extend the absolute lifetime", + ); + assert!(last_served >= pinned_at); + + // Once that deadline passes the pin goes, even though it was just touched. The + // peer can only get it back while the checkpoint is still in the registry — a + // pruned one is gone for good, which is the retention this bounds. + let Some(long_ago) = Instant::now().checked_sub(SERVING_PIN_MAX_LIFETIME * 2) else { + return; + }; + manager.backdate_pin(10, Some(long_ago), None); + assert!( + manager.pinned_checkpoint(10).is_none(), + "an over-long pin must expire despite continued activity", + ); + } + + /// A peer that keeps touching new heights as pruning advances must not be able to + /// hold an unbounded number of checkpoint directories back from deletion. + #[test] + fn serving_pins_are_capped() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let manager = SnapshotManager::new(); + + // The cap tracks how many snapshots the node retains, plus slack for transfers + // that started before their checkpoint aged out. + let max_pins = max_serving_pins(3); + assert_eq!(max_pins, 3 + SERVING_PIN_SLACK); + + for height in 0..(max_pins as u64 + 4) { + manager.pin_for_serving(height, checkpoint_in(&dir, height), max_pins); + } + + assert_eq!(manager.pinned_count(), max_pins); + assert!( + manager.pinned_checkpoint(0).is_none(), + "the least recently served pin must be evicted first", + ); + assert!(manager.pinned_checkpoint(max_pins as u64 + 3).is_some()); + } + + #[test] + fn supported_wire_versions_include_the_version_platform_versions_stamp() { + use dpp::version::PlatformVersion; + // Every platform version stamps its state_sync.protocol_version on the + // snapshots it offers; the supported set must accept what we serve. + for platform_version in dpp::version::ALL_VERSIONS + .map(PlatformVersion::get) + .filter_map(Result::ok) + { + assert!( + SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS + .contains(&platform_version.drive_abci.state_sync.protocol_version), + "platform version {} stamps unsupported state sync wire version {}", + platform_version.protocol_version, + platform_version.drive_abci.state_sync.protocol_version + ); + } + } +} diff --git a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs index e6aa26b0fce..7d7b75bc9c9 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs @@ -11,6 +11,7 @@ use super::*; use crate::query::tests::{setup_platform, store_data_contract, store_document}; +use crate::test::helpers::fast_forward_to_block::TEST_BLOCK_SIGNATURE; use dapi_grpc::platform::v0::get_documents_request::get_documents_request_v1::{ select as v1_select, Select as V1Select, Start as V1Start, }; @@ -4549,7 +4550,7 @@ mod time_range_proof_verification { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), diff --git a/packages/rs-drive-abci/src/query/response_metadata/v0/mod.rs b/packages/rs-drive-abci/src/query/response_metadata/v0/mod.rs index 19621528234..461e651a32a 100644 --- a/packages/rs-drive-abci/src/query/response_metadata/v0/mod.rs +++ b/packages/rs-drive-abci/src/query/response_metadata/v0/mod.rs @@ -1,3 +1,4 @@ +use crate::abci::AbciError; use crate::error::Error; use crate::platform_types::platform::Platform; use crate::platform_types::platform_state::PlatformState; @@ -7,6 +8,44 @@ use dapi_grpc::platform::v0::{Proof, ResponseMetadata}; use drive::error::drive::DriveError; use drive::util::grove_operations::GroveDBToUse; +impl Platform { + /// Refuses to build a proof from a state that has no block proof metadata. + /// + /// A state restored via state sync has an all-zero block id hash and quorum signature + /// at the snapshot height: the reduced platform state is written into grovedb BEFORE + /// the block's root hash exists, and the block's commit signature signs that root + /// hash, so the signature can never be part of the state it signs. + /// `rs-drive-proof-verifier` (correctly) rejects an all-zero signature, meaning a + /// proof built from such a state could never authenticate — refuse it with a + /// retryable error instead. The first block finalized after the restore stores real + /// metadata and reopens proof serving. + /// + /// Height 0 is exempt: a chain that has not committed a block yet has no signature + /// either, which predates state sync and is left as is. + fn ensure_block_proof_metadata_is_available(&self, state: &PlatformState) -> Result<(), Error> { + // A test chain running with block signing disabled finalizes every block with an + // all-zero signature; its proofs were never verifiable, and gating them would + // break the strategy test harness's proof plumbing checks. + #[cfg(feature = "testing-config")] + if !self.config.testing_configs.block_signing { + return Ok(()); + } + + if state.last_committed_block_height() > 0 + && state.last_committed_block_signature() == [0u8; 96] + { + return Err(AbciError::StateSyncProofMetadataUnavailable(format!( + "the state at height {} was restored via state sync and its block signature \ + only becomes known when the next block is finalized; retry shortly, or \ + repeat the query without requesting a proof", + state.last_committed_block_height() + )) + .into()); + } + Ok(()) + } +} + impl Platform { /// Returns response metadata for the given GroveDB that was used. /// @@ -46,6 +85,7 @@ impl Platform { ) -> Result<(CheckpointUsed, Proof), Error> { match grovedb_to_use { GroveDBToUse::Current => { + self.ensure_block_proof_metadata_is_available(platform_state)?; let proof = Proof { grovedb_proof: proof, quorum_hash: platform_state.last_committed_quorum_hash().to_vec(), @@ -74,6 +114,7 @@ impl Platform { })? .clone(); + self.ensure_block_proof_metadata_is_available(&checkpoint_state)?; let proof = Proof { grovedb_proof: proof, quorum_hash: checkpoint_state.last_committed_quorum_hash().to_vec(), @@ -95,6 +136,7 @@ impl Platform { })? .clone(); + self.ensure_block_proof_metadata_is_available(&checkpoint_state)?; let proof = Proof { grovedb_proof: proof, quorum_hash: checkpoint_state.last_committed_quorum_hash().to_vec(), @@ -108,3 +150,85 @@ impl Platform { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::block::block_info::BlockInfo; + use dpp::block::extended_block_info::v0::ExtendedBlockInfoV0; + use dpp::block::extended_block_info::ExtendedBlockInfo; + + fn block_info_with_signature(height: u64, signature: [u8; 96]) -> ExtendedBlockInfo { + ExtendedBlockInfo::V0(ExtendedBlockInfoV0 { + basic_info: BlockInfo { + time_ms: 1_000_000, + height, + core_height: 42, + epoch: Default::default(), + }, + app_hash: [1u8; 32], + quorum_hash: [2u8; 32], + block_id_hash: [3u8; 32], + proposer_pro_tx_hash: [4u8; 32], + signature, + round: 0, + }) + } + + /// A state restored via state sync stores an all-zero block signature until the next + /// block finalizes; a proof built from it can never authenticate (the verifier + /// rejects an all-zero signature), so it must be refused rather than served. + #[test] + fn should_refuse_a_proof_from_a_state_without_block_proof_metadata() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let mut state = platform.state.load().as_ref().clone(); + state.set_last_committed_block_info(Some(block_info_with_signature(10, [0u8; 96]))); + + let result = platform.response_proof_v0(&state, vec![], GroveDBToUse::Current); + let error = result.expect_err("a zero-signature state must not produce a proof"); + assert!( + matches!( + error, + Error::Abci(AbciError::StateSyncProofMetadataUnavailable(_)) + ), + "expected StateSyncProofMetadataUnavailable, got: {error}" + ); + } + + /// A normally finalized block always carries a real signature; proofs must be served. + #[test] + fn should_serve_a_proof_from_a_state_with_block_proof_metadata() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let mut state = platform.state.load().as_ref().clone(); + state.set_last_committed_block_info(Some(block_info_with_signature(10, [5u8; 96]))); + + let (_, proof) = platform + .response_proof_v0(&state, vec![], GroveDBToUse::Current) + .expect("a signed state must produce a proof"); + assert_eq!(proof.signature, vec![5u8; 96]); + assert_eq!(proof.block_id_hash, vec![3u8; 32]); + } + + /// A chain that has not committed a block yet has no signature for anyone — that + /// predates state sync and stays as it was. + #[test] + fn should_leave_the_pre_genesis_state_exempt() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let state = platform.state.load(); + assert_eq!(state.last_committed_block_height(), 0, "sanity: no blocks"); + + platform + .response_proof_v0(&state, vec![], GroveDBToUse::Current) + .expect("the pre-genesis state must remain servable"); + } +} diff --git a/packages/rs-drive-abci/src/query/service.rs b/packages/rs-drive-abci/src/query/service.rs index 3c6ce47b5a3..ac59915dcdf 100644 --- a/packages/rs-drive-abci/src/query/service.rs +++ b/packages/rs-drive-abci/src/query/service.rs @@ -126,11 +126,16 @@ impl QueryService { // that query is executed only after/before both states are updated. let mut needs_restart = false; + // The wait budget must survive iterations of the loop below: declared + // inside it, the counter was reset on every pass and the 1 second + // timeout could never fire, so a query arriving while the two states + // disagreed would spin here forever instead of restarting. + let mut counter = 0; + loop { let committed_block_height_guard = platform .committed_block_height_guard .load(Ordering::Relaxed); - let mut counter = 0; if platform_state.last_committed_block_height() == committed_block_height_guard { break; @@ -1001,7 +1006,15 @@ fn query_error_into_status(error: QueryError) -> Status { } fn error_into_status(error: Error) -> Status { - Status::internal(format!("query: {}", error)) + match error { + // Not a server fault: the state was restored via state sync and the block proof + // metadata arrives with the first block finalized after the restore. UNAVAILABLE + // tells clients to retry (or drop the proof request) rather than report a bug. + Error::Abci(crate::abci::AbciError::StateSyncProofMetadataUnavailable(message)) => { + Status::unavailable(message) + } + error => Status::internal(format!("query: {}", error)), + } } fn validate_path_elements_request(request: &GetPathElementsRequest) -> Result<(), Status> { diff --git a/packages/rs-drive-abci/src/server.rs b/packages/rs-drive-abci/src/server.rs index 3baf33f5c2a..bf8d1b8e3e1 100644 --- a/packages/rs-drive-abci/src/server.rs +++ b/packages/rs-drive-abci/src/server.rs @@ -5,6 +5,7 @@ use crate::abci::app::CheckTxAbciApplication; use crate::abci::app::ConsensusAbciApplication; use crate::config::PlatformConfig; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::{SnapshotManager, SERVING_PIN_SWEEP_INTERVAL}; use crate::query::QueryService; use crate::rpc::core::DefaultCoreRPC; use std::sync::Arc; @@ -31,8 +32,30 @@ pub fn start( ) .expect("failed to open check tx core rpc"); - let check_tx_service = - CheckTxAbciApplication::new(Arc::clone(&platform), Arc::new(check_tx_core_rpc)); + // The snapshot manager pins the checkpoints being served to state-syncing peers. + // It lives with the gRPC application below (which answers ListSnapshots and + // LoadSnapshotChunk), but that application never sees blocks, so nothing on its + // request path would ever release the pin of an ABANDONED transfer — the peer + // simply stops asking. A shared handle and a timer task make expiry autonomous. + let snapshot_manager = Arc::new(SnapshotManager::new()); + + let serving_pin_sweep_cancel = cancel.clone(); + let serving_pin_sweep_manager = Arc::clone(&snapshot_manager); + runtime.spawn(async move { + let mut interval = tokio::time::interval(SERVING_PIN_SWEEP_INTERVAL); + loop { + tokio::select! { + _ = serving_pin_sweep_cancel.cancelled() => break, + _ = interval.tick() => serving_pin_sweep_manager.release_expired_pins(), + } + } + }); + + let check_tx_service = CheckTxAbciApplication::new( + Arc::clone(&platform), + Arc::new(check_tx_core_rpc), + snapshot_manager, + ); let grpc_server = dapi_grpc::tonic::transport::Server::builder() .add_service( diff --git a/packages/rs-drive-abci/src/test/helpers/fast_forward_to_block.rs b/packages/rs-drive-abci/src/test/helpers/fast_forward_to_block.rs index e991b965860..06d2880df84 100644 --- a/packages/rs-drive-abci/src/test/helpers/fast_forward_to_block.rs +++ b/packages/rs-drive-abci/src/test/helpers/fast_forward_to_block.rs @@ -17,6 +17,12 @@ use drive::drive::credit_pools::operations::update_unpaid_epoch_index_operation; use platform_version::version::PlatformVersion; use std::sync::Arc; +/// Placeholder block signature for hand-built `ExtendedBlockInfo` test fixtures. +/// +/// Any non-zero value works: an all-zero signature marks a state restored via state +/// sync, from which proofs are refused until the next block finalizes. +pub(crate) const TEST_BLOCK_SIGNATURE: [u8; 96] = [1u8; 96]; + pub(crate) fn fast_forward_to_block( platform: &TempPlatform, time_ms: u64, @@ -51,7 +57,7 @@ pub(crate) fn fast_forward_to_block( quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), diff --git a/packages/rs-drive-abci/src/utils/mod.rs b/packages/rs-drive-abci/src/utils/mod.rs index bfebd6dcd62..05bbdf32e46 100644 --- a/packages/rs-drive-abci/src/utils/mod.rs +++ b/packages/rs-drive-abci/src/utils/mod.rs @@ -4,5 +4,6 @@ mod spawn; pub(crate) use replay::is_historical_block; pub use serialization::from_opt_str_or_number; +pub use serialization::from_str_or_native; pub use serialization::from_str_or_number; pub use spawn::spawn_blocking_task_with_name_if_supported; diff --git a/packages/rs-drive-abci/src/utils/serialization.rs b/packages/rs-drive-abci/src/utils/serialization.rs index 8259ff1dce3..cc5b965d49f 100644 --- a/packages/rs-drive-abci/src/utils/serialization.rs +++ b/packages/rs-drive-abci/src/utils/serialization.rs @@ -13,6 +13,29 @@ where s.parse::().map_err(Error::custom) } +/// Deserialize a value from a string (as provided by envy, where every value is a +/// string) or from its native representation (as in JSON round trips). +pub fn from_str_or_native<'de, D, T>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, + T: serde::Deserialize<'de> + std::str::FromStr, + ::Err: std::fmt::Display, +{ + use serde::de::Error; + + #[derive(Deserialize)] + #[serde(untagged)] + enum NativeOrString { + Native(T), + String(String), + } + + match NativeOrString::::deserialize(deserializer)? { + NativeOrString::Native(value) => Ok(value), + NativeOrString::String(s) => s.parse::().map_err(Error::custom), + } +} + /// Deserialize a value from an optional string or a number pub fn from_opt_str_or_number<'de, D, T>(deserializer: D) -> Result, D::Error> where diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs index 1e963cb1cbd..3a6b59a8e27 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs @@ -11,6 +11,8 @@ mod process_proposal_collision_tests; mod required_since_update_tests; // TODO: re-enable once OperationType shielded variants are implemented // mod shielded_tests; +mod state_sync_sentinel_tests; +pub(crate) mod state_sync_tests; mod token_tests; mod top_up_tests; mod update_identities_tests; diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/process_proposal_collision_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/process_proposal_collision_tests.rs index b2fe58a7222..327ca11e15c 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/process_proposal_collision_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/process_proposal_collision_tests.rs @@ -566,4 +566,52 @@ mod tests { "the cached context must be the one prepare proposal built" ); } + + /// CONSENSUS PIN: the app hash must not depend on the consensus round. + /// + /// Tenderdash re-proposes a block that reached a prevote majority but did not commit + /// with the SAME header at a later round, and re-runs ProcessProposal for every round, + /// requiring the returned app hash to equal the header's. If anything round-specific + /// reached the replicated state (as the reduced platform state written by + /// `run_block_proposal` v1 once did), every validator would reject the re-proposal + /// and the chain would halt at that height. + #[tokio::test] + async fn process_proposal_of_the_same_block_at_a_later_round_must_return_the_same_app_hash() { + let config = config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + + let outcome = run_chain_for_strategy( + &mut platform, + 5, + strategy(), + config, + 7, + &mut None, + &mut None, + ) + .await; + + let at_round_0 = next_block_request(&outcome, 1, [0x42u8; 32], 0); + let mut at_round_3 = at_round_0.clone(); + at_round_3.round = 3; + + let response_round_0 = outcome + .abci_app + .process_proposal(at_round_0) + .expect("the block processes at round 0"); + assert_eq!(response_round_0.status, ProposalStatus::Accept as i32); + + let response_round_3 = outcome + .abci_app + .process_proposal(at_round_3) + .expect("the same block processes again at round 3"); + assert_eq!(response_round_3.status, ProposalStatus::Accept as i32); + + assert_eq!( + response_round_0.app_hash, response_round_3.app_hash, + "the same block re-proposed at a later round must produce the same app hash" + ); + } } diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs new file mode 100644 index 00000000000..a0a43f84789 --- /dev/null +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs @@ -0,0 +1,578 @@ +//! State sync QA: the restore sentinel and the never-wedge guarantee. +//! +//! A state sync restore destroys the node's database before it rebuilds it, and the +//! rebuild is not atomic with the platform state that has to describe it. Two things can +//! therefore leave a node holding a database its platform state knows nothing about: +//! +//! * the process dies between `commit_session` and `reconstruct_platform_state`; +//! * the restored snapshot turns out to be unusable (a pre-v15 snapshot with no reduced +//! platform state, or one that fails verification) — which any peer can cause. +//! +//! Both used to wedge the node permanently, because the `info` handler panics on an +//! app-hash mismatch and restarting reloads exactly the state that causes the panic. The +//! fix is a sentinel file written next to the database before the wipe, cleared only when +//! the node is self-consistent again, plus a rejection path that wipes back to a clean +//! slate instead of returning an error. +//! +//! Nothing here asserts that a restore SUCCEEDS — the tests that do live in +//! `state_sync_tests`. What is asserted is that a restore which does not succeed leaves +//! a recoverable node. + +#[cfg(test)] +mod tests { + use crate::execution::run_chain_for_strategy; + use crate::strategy::{ChainExecutionOutcome, NetworkStrategy}; + use crate::test_cases::state_sync_tests::tests::{ + install_reconstruction_core_mocks, recent_start_time_ms, sync_snapshot, SnapshotSyncOutcome, + }; + use dpp::version::v15::PROTOCOL_VERSION_15; + use dpp::version::PlatformVersion; + use drive_abci::abci::app::FullAbciApplication; + use drive_abci::config::{ + ChainLockConfig, ExecutionConfig, InstantLockConfig, PlatformConfig, PlatformTestConfig, + ValidatorSetConfig, + }; + use drive_abci::platform_types::platform::Platform; + use drive_abci::platform_types::platform_state::PlatformStateV0Methods; + use drive_abci::platform_types::snapshot::{ + encode_snapshot_metadata, restore_sentinel_exists, write_restore_sentinel, + RESTORE_IN_PROGRESS_FILE_NAME, + }; + use drive_abci::rpc::core::MockCoreRPCLike; + use drive_abci::test::helpers::setup::{TempPlatform, TestPlatformBuilder}; + use strategy_tests::frequency::Frequency; + use strategy_tests::{IdentityInsertInfo, StartAddresses, StartIdentities, Strategy}; + use tenderdash_abci::proto::abci as proto; + use tenderdash_abci::proto::abci::response_offer_snapshot; + use tenderdash_abci::Application; + + const SOURCE_CHAIN_BLOCKS: u64 = 6; + const SOURCE_CHAIN_SEED: u64 = 15; + + fn sentinel_platform_config() -> PlatformConfig { + let mut testing_configs = PlatformTestConfig::default_minimal_verifications(); + testing_configs.disable_checkpoints = false; + testing_configs.store_platform_state = true; + + let mut config = PlatformConfig { + validator_set: ValidatorSetConfig::default_100_67(), + chain_lock: ChainLockConfig::default_100_67(), + instant_lock: InstantLockConfig::default_100_67(), + execution: ExecutionConfig { + verify_sum_trees: true, + ..ExecutionConfig::default() + }, + block_spacing_ms: 3000, + testing_configs, + ..Default::default() + }; + config.abci.state_sync.snapshots_enabled = true; + config.abci.state_sync.snapshots_frequency_seconds = 1; + config.abci.state_sync.max_num_snapshots = 3; + config + } + + fn sentinel_strategy() -> NetworkStrategy { + NetworkStrategy { + strategy: Strategy { + start_contracts: vec![], + operations: vec![], + start_identities: StartIdentities::default(), + start_addresses: StartAddresses::default(), + identity_inserts: IdentityInsertInfo { + frequency: Frequency { + times_per_block_range: 1..3, + chance_per_block: None, + }, + ..Default::default() + }, + identity_contract_nonce_gaps: None, + signer: None, + }, + total_hpmns: 100, + extra_normal_mns: 0, + validator_quorum_count: 24, + chain_lock_quorum_count: 24, + upgrading_info: None, + proposer_strategy: Default::default(), + rotate_quorums: false, + failure_testing: None, + query_testing: None, + verify_state_transition_results: false, + start_time_ms: recent_start_time_ms(), + ..Default::default() + } + } + + fn root_hash(platform: &Platform) -> [u8; 32] { + platform + .drive + .grove + .root_hash(None, &PlatformVersion::latest().drive.grove_version) + .unwrap() + .expect("root hash") + } + + fn info_request() -> proto::RequestInfo { + proto::RequestInfo { + version: tenderdash_abci::proto::meta::TENDERDASH_VERSION.to_string(), + block_version: 0, + p2p_version: 0, + abci_version: tenderdash_abci::proto::meta::ABCI_VERSION.to_string(), + } + } + + /// Models process death: drops the `Platform` (releasing grovedb's lock and every + /// in-memory session, cache and platform state) and re-opens the SAME directory, which + /// is what a restarted drive-abci does. Only what was durably written survives. + fn restart( + target: TempPlatform, + config: &PlatformConfig, + ) -> TempPlatform { + let TempPlatform { + platform, tempdir, .. + } = target; + drop(platform); + TempPlatform::open_with_tempdir(tempdir, config.clone()) + } + + /// Calling `info` must not panic. The handler panics on an app-hash mismatch between + /// the platform state and grovedb, which is the exact shape of the wedge, so "did it + /// panic" is the property under test rather than the returned value. + fn info_does_not_panic(platform: &TempPlatform) -> bool { + let app = FullAbciApplication::new(platform); + let previous_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| app.info(info_request()))); + std::panic::set_hook(previous_hook); + result.is_ok() + } + + /// `offer_snapshot` must record the sentinel BEFORE it wipes, so there is no window in + /// which the database has been destroyed and nothing says so. + #[tokio::test] + async fn offer_snapshot_records_the_restore_sentinel_before_wiping() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let app = outcome.abci_app; + let db_path = app.platform.config.db_path.clone(); + + assert!( + !restore_sentinel_exists(&db_path), + "a node that never state-synced must not carry the sentinel" + ); + + let response = app + .offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1000, + version: 1, + hash: vec![7u8; 32], + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), + }), + app_hash: vec![7u8; 32], + }) + .expect("the offer must be accepted"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + + assert!( + restore_sentinel_exists(&db_path), + "accepting an offer wipes the database, so it must first record that a restore \ + is in progress" + ); + // The sentinel is a plain file NEXT TO the database, not aux storage: `wipe()` + // clears the aux column family too, so a sentinel stored there would be destroyed + // by the very wipe it exists to survive. + assert!( + db_path.join(RESTORE_IN_PROGRESS_FILE_NAME).is_file(), + "the sentinel must live outside everything grovedb wipes" + ); + } + + /// A wipe must also drop the CHECKPOINT REGISTRY, not just the value caches. + /// + /// `drive.checkpoints` is populated by `Drive::open` and is what `list_snapshots` + /// serves to peers. It is not a cache that merely goes stale: left in place across a + /// wipe, a node that discarded a chain would keep advertising snapshots of it, and a + /// peer state-syncing from those would restore a chain this node no longer has and + /// cannot vouch for. + #[tokio::test] + async fn a_wiped_node_stops_serving_snapshots_of_the_discarded_chain() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let app = outcome.abci_app; + + assert!( + !app.list_snapshots(Default::default()) + .expect("should list snapshots") + .snapshots + .is_empty(), + "sanity: the chain must have produced servable snapshots to begin with" + ); + + // Accepting an offer wipes the database out from under those checkpoints. + app.offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1000, + version: 1, + hash: vec![7u8; 32], + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), + }), + app_hash: vec![7u8; 32], + }) + .expect("the offer must be accepted"); + + assert!( + app.list_snapshots(Default::default()) + .expect("should list snapshots") + .snapshots + .is_empty(), + "after a wipe the node must stop advertising snapshots of the chain it just \ + discarded — otherwise a peer would state-sync from state this node no longer has" + ); + assert!( + app.platform.drive.checkpoints.load().is_empty(), + "the checkpoint registry itself must be cleared, not just filtered at serve time" + ); + } + + /// A rejected offer must not record a sentinel — nothing was wiped, so nothing needs + /// recovering. Without this, any peer could make a healthy node wipe itself on the next + /// restart just by offering a snapshot in a format it cannot speak. + #[tokio::test] + async fn a_rejected_offer_records_no_sentinel() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let app = outcome.abci_app; + let db_path = app.platform.config.db_path.clone(); + + let response = app + .offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1000, + version: u32::MAX, + hash: vec![7u8; 32], + metadata: vec![], + }), + app_hash: vec![7u8; 32], + }) + .expect("an unsupported version must be answered, not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::RejectFormat) + ); + assert!( + !restore_sentinel_exists(&db_path), + "a rejected offer wipes nothing and must leave no sentinel behind" + ); + } + + /// The startup recovery itself: a node whose sentinel is still present comes up EMPTY + /// rather than crash-looping. + /// + /// The database here is deliberately a healthy, fully populated chain — the strongest + /// form of "grovedb holds state the platform state will not describe". Recovery must + /// throw it away, because there is no way to tell how far an interrupted restore got. + #[tokio::test] + async fn a_node_restarting_mid_restore_wipes_and_comes_up_empty() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let db_path = outcome.abci_app.platform.config.db_path.clone(); + assert_eq!( + outcome + .abci_app + .platform + .state + .load() + .last_committed_block_height(), + SOURCE_CHAIN_BLOCKS + ); + drop(outcome); + + // A restore was in progress when the process died. + write_restore_sentinel(&db_path, &[9u8; 32], 1000).expect("write sentinel"); + + let restarted = restart(platform, &config); + + assert_eq!( + restarted.state.load().last_committed_block_height(), + 0, + "an unfinished restore must not leave the node claiming a height it cannot back up" + ); + assert_eq!( + root_hash(&restarted.platform), + [0u8; 32], + "the database must have been wiped to an empty, self-consistent state" + ); + assert!( + !restore_sentinel_exists(&db_path), + "once the node is empty it is self-consistent again, so the sentinel is cleared" + ); + assert!( + info_does_not_panic(&restarted), + "THE WHOLE POINT: the info handshake must succeed, so the node can be offered \ + another snapshot or fall back to block sync instead of crash-looping" + ); + } + + /// The complement, and the regression that matters most: a node WITHOUT a sentinel + /// must never be wiped. If startup recovery ever fires unconditionally it would + /// silently destroy every node's chain on restart. + #[tokio::test] + async fn a_normal_restart_keeps_its_state() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let db_path = outcome.abci_app.platform.config.db_path.clone(); + let healthy_root_hash = root_hash(outcome.abci_app.platform); + drop(outcome); + + assert!(!restore_sentinel_exists(&db_path)); + let restarted = restart(platform, &config); + + assert_eq!( + restarted.state.load().last_committed_block_height(), + SOURCE_CHAIN_BLOCKS, + "a normal restart must come back at the tip" + ); + assert_eq!( + root_hash(&restarted.platform), + healthy_root_hash, + "a normal restart must not touch the database" + ); + assert!(info_does_not_panic(&restarted)); + } + + /// The block-sync arm of the recovery. If every offered snapshot is rejected, + /// Tenderdash gives up on state sync and block-syncs from genesis. `init_chain` is + /// where the node becomes self-consistent again, so it must clear a sentinel left over + /// from the abandoned restore — otherwise the NEXT restart would wipe a perfectly good + /// chain. + #[tokio::test] + async fn init_chain_clears_a_sentinel_left_by_an_abandoned_restore() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let db_path = platform.platform.config.db_path.clone(); + + // An abandoned restore: the marker is present and the database is empty. + write_restore_sentinel(&db_path, &[9u8; 32], 1000).expect("write sentinel"); + + // Block sync from genesis, which begins with init_chain. + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + + assert_eq!( + outcome + .abci_app + .platform + .state + .load() + .last_committed_block_height(), + SOURCE_CHAIN_BLOCKS, + "the node must block-sync normally after an abandoned restore" + ); + assert!( + !restore_sentinel_exists(&db_path), + "init_chain makes the node self-consistent, so it must clear the sentinel — \ + otherwise the next restart would wipe this chain" + ); + drop(outcome); + + // And prove it: a restart keeps the block-synced chain. + let restarted = restart(platform, &config); + assert_eq!( + restarted.state.load().last_committed_block_height(), + SOURCE_CHAIN_BLOCKS, + "the chain built after an abandoned restore must survive a restart" + ); + } + + /// End to end for the remotely-triggerable case: a peer offers a snapshot this node + /// cannot use, and the node must end up able to sync rather than wedged. + /// + /// The snapshot here is a pre-v15 one (a v14 chain's checkpoint, which carries no + /// reduced platform state). Nothing stops a peer from advertising it: `proto::Snapshot` + /// carries a height, a wire version and a hash, and no protocol version at all. + /// + /// This test is pin-agnostic on purpose. With grovedb #840 the refusal comes from the + /// missing reduced platform state; at the unpatched revision the sum-tree defect makes + /// the post-restore verification fail first. Either way the snapshot is refused AFTER + /// the session was committed, which is precisely the path that has to leave the node + /// recoverable. + #[tokio::test] + async fn an_unusable_snapshot_leaves_the_node_able_to_sync() { + let config = sentinel_platform_config(); + + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .with_initial_protocol_version(14) + .build_with_mock_rpc(); + let ChainExecutionOutcome { + abci_app: source_app, + proposers, + validator_quorums, + .. + } = run_chain_for_strategy( + &mut source_platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + + let (height, checkpoint) = { + let checkpoints = source_app.platform.drive.checkpoints.load(); + let (height, info) = checkpoints + .last_key_value() + .expect("at least one checkpoint"); + (*height, std::sync::Arc::clone(&info.checkpoint)) + }; + let platform_version = PlatformVersion::latest(); + let checkpoint_root = checkpoint + .grove_db + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("checkpoint root hash"); + let forged_snapshot = proto::Snapshot { + height, + version: platform_version.drive_abci.state_sync.protocol_version as u32, + hash: checkpoint_root.to_vec(), + metadata: encode_snapshot_metadata(platform_version.protocol_version), + }; + + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + proposers + .iter() + .map(|proposer| proposer.masternode.clone()) + .collect(), + &validator_quorums, + ); + let db_path = target_platform.platform.config.db_path.clone(); + + { + let target_app = FullAbciApplication::new(&target_platform); + let outcome = sync_snapshot(&source_app, &target_app, &forged_snapshot, false) + .expect("an unusable snapshot must be answered, not errored"); + assert_eq!( + outcome, + SnapshotSyncOutcome::Rejected, + "an unusable snapshot must be answered with REJECT_SNAPSHOT so Tenderdash \ + tries the next one instead of aborting state sync" + ); + + // The node wiped itself back to a clean slate rather than keeping state it + // cannot use... + assert_ne!( + root_hash(&target_platform.platform).to_vec(), + forged_snapshot.hash, + "the refused snapshot must not be left on disk" + ); + assert_eq!( + root_hash(&target_platform.platform), + [0u8; 32], + "the refusal must leave an empty database" + ); + // ...and the sentinel stays, because the in-memory platform state may still + // describe the chain the offer wiped. Whatever happens next resolves it. + assert!( + restore_sentinel_exists(&db_path), + "the node is empty but not yet provably consistent, so the marker stays \ + until a restore succeeds, an init_chain runs, or a restart wipes" + ); + } + + // A restart is the worst case, and it recovers. + let restarted = restart(target_platform, &config); + assert_eq!(restarted.state.load().last_committed_block_height(), 0); + assert_eq!(root_hash(&restarted.platform), [0u8; 32]); + assert!( + !restore_sentinel_exists(&db_path), + "startup recovery leaves the node self-consistent and clears the marker" + ); + assert!( + info_does_not_panic(&restarted), + "THE FIX: a peer offering an unusable snapshot must not be able to wedge this \ + node. Before the fix, info panicked here and drive-abci crash-looped." + ); + } +} diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs new file mode 100644 index 00000000000..0480ee8b379 --- /dev/null +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -0,0 +1,836 @@ +//! Two-instance ABCI state sync integration tests: a source chain serves snapshots from +//! its checkpoint registry and a fresh target restores one chunk by chunk, then +//! reconstructs its platform state. + +#[cfg(test)] +pub(crate) mod tests { + use crate::execution::run_chain_for_strategy; + use crate::strategy::{ChainExecutionOutcome, NetworkStrategy}; + use dpp::dashcore::hashes::Hash; + use dpp::dashcore::{BlockHash, QuorumHash}; + use dpp::dashcore_rpc::dashcore_rpc_json::{ + ExtendedQuorumDetails, MasternodeListDiff, MasternodeListItem, QuorumInfoResult, + }; + use dpp::dashcore_rpc::json::{ExtendedQuorumListResult, QuorumType}; + use dpp::version::v15::PROTOCOL_VERSION_15; + use dpp::version::PlatformVersion; + use drive_abci::abci::app::FullAbciApplication; + use drive_abci::config::{ + ChainLockConfig, ExecutionConfig, InstantLockConfig, PlatformConfig, PlatformTestConfig, + ValidatorSetConfig, + }; + use drive_abci::mimic::test_quorum::TestQuorumInfo; + use drive_abci::platform_types::platform::Platform; + use drive_abci::platform_types::platform_state::PlatformStateV0Methods; + use drive_abci::platform_types::snapshot::encode_snapshot_metadata; + use drive_abci::rpc::core::MockCoreRPCLike; + use drive_abci::test::helpers::setup::{TempPlatform, TestPlatformBuilder}; + use std::collections::{BTreeMap, HashMap, VecDeque}; + use strategy_tests::frequency::Frequency; + use strategy_tests::{IdentityInsertInfo, StartAddresses, StartIdentities, Strategy}; + use tenderdash_abci::proto::abci as proto; + use tenderdash_abci::proto::abci::{response_apply_snapshot_chunk, response_offer_snapshot}; + use tenderdash_abci::Application; + + /// A first-block time near the wall clock. Checkpoints are only taken for blocks + /// younger than ten minutes (`is_historical_block`), so a source chain that starts + /// at the fixed 2023 genesis time never produces a snapshot. + pub(crate) fn recent_start_time_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock is before the unix epoch") + .as_millis() as u64 + } + + /// A quiet chain with a trickle of identity inserts, no masternode churn and no + /// quorum rotation, so the target's from-scratch Core re-derivation sees exactly + /// the same masternodes and quorums the source chain ran with. + fn state_sync_network_strategy() -> NetworkStrategy { + NetworkStrategy { + strategy: Strategy { + start_contracts: vec![], + operations: vec![], + start_identities: StartIdentities::default(), + start_addresses: StartAddresses::default(), + identity_inserts: IdentityInsertInfo { + frequency: Frequency { + times_per_block_range: 1..2, + chance_per_block: None, + }, + ..Default::default() + }, + identity_contract_nonce_gaps: None, + signer: None, + }, + total_hpmns: 100, + extra_normal_mns: 0, + validator_quorum_count: 24, + chain_lock_quorum_count: 24, + upgrading_info: None, + proposer_strategy: Default::default(), + rotate_quorums: false, + failure_testing: None, + query_testing: None, + verify_state_transition_results: false, + start_time_ms: recent_start_time_ms(), + ..Default::default() + } + } + + /// Snapshot serving on with a 1s frequency (every 3s block crosses the boundary, + /// so every block after the first creates a checkpoint), keeping 3 checkpoints. + fn state_sync_platform_config() -> PlatformConfig { + let mut testing_configs = PlatformTestConfig::default_minimal_verifications(); + testing_configs.disable_checkpoints = false; + testing_configs.store_platform_state = true; + + let mut config = PlatformConfig { + validator_set: ValidatorSetConfig::default_100_67(), + chain_lock: ChainLockConfig::default_100_67(), + instant_lock: InstantLockConfig::default_100_67(), + execution: ExecutionConfig { + verify_sum_trees: true, + ..ExecutionConfig::default() + }, + block_spacing_ms: 3000, + testing_configs, + ..Default::default() + }; + config.abci.state_sync.snapshots_enabled = true; + config.abci.state_sync.snapshots_frequency_seconds = 1; + config.abci.state_sync.max_num_snapshots = 3; + config + } + + /// Installs on a fresh target the Core RPC answers its platform state + /// reconstruction will ask for: the full masternode list (the target requests it + /// from scratch, base height None) and the same quorums the source ran with. + pub(crate) fn install_reconstruction_core_mocks( + platform: &mut Platform, + masternodes: Vec, + validator_quorums: &BTreeMap, + ) { + platform + .core_rpc + .expect_get_protx_diff_with_masternodes() + .returning(move |base_block, block| { + assert!( + base_block.is_none(), + "state reconstruction must request the full masternode list from scratch" + ); + Ok(MasternodeListDiff { + base_height: 0, + block_height: block, + added_mns: masternodes.clone(), + removed_mns: vec![], + updated_mns: vec![], + }) + }); + + let quorum_details: Vec<(QuorumHash, ExtendedQuorumDetails)> = validator_quorums + .keys() + .map(|quorum_hash| { + ( + *quorum_hash, + ExtendedQuorumDetails { + creation_height: 0, + quorum_index: None, + mined_block_hash: BlockHash::all_zeros(), + num_valid_members: 0, + health_ratio: 0.0, + }, + ) + }) + .collect(); + platform + .core_rpc + .expect_get_quorum_listextended() + .returning(move |_| { + Ok(ExtendedQuorumListResult { + quorums_by_type: HashMap::from([( + QuorumType::Llmq100_67, + quorum_details.clone().into_iter().collect(), + )]), + }) + }); + + let quorum_infos: HashMap = validator_quorums + .iter() + .map(|(quorum_hash, test_quorum_info)| (*quorum_hash, test_quorum_info.into())) + .collect(); + platform.core_rpc.expect_get_quorum_info().returning( + move |_, quorum_hash: &QuorumHash, _| { + Ok(quorum_infos + .get::(quorum_hash) + .unwrap_or_else(|| { + panic!("expected to get quorum {}", hex::encode(quorum_hash)) + }) + .clone()) + }, + ); + } + + /// How a snapshot transfer ended. + /// + /// `Rejected` is not an error: the target restored the snapshot, found it unusable, + /// wiped itself back to a clean slate and asked Tenderdash for a different one. The + /// driver reports it so tests can tell a clean refusal from a transport failure. + #[derive(Debug, PartialEq, Eq)] + pub(crate) enum SnapshotSyncOutcome { + /// The target restored and accepted the snapshot. + Completed, + /// The target answered REJECT_SNAPSHOT; Tenderdash would move on to the next one. + Rejected, + } + + /// Drives the chunk transfer loop between a serving app and a restoring app, + /// modeled on grovedb's run_sync driver: start from the root chunk (id == app + /// hash) and keep requesting whatever the target asks for next. + /// + /// When `tamper_with_first_chunk` is set, the first served chunk is corrupted to + /// prove the target answers RETRY_SNAPSHOT (banning the sender) instead of killing + /// the session: grovedb invalidates its session on a failed chunk, so the driver + /// handles that the way Tenderdash would, by re-offering the same snapshot and + /// restarting the transfer. + pub(crate) fn sync_snapshot( + source_app: &FullAbciApplication, + target_app: &FullAbciApplication, + snapshot: &proto::Snapshot, + tamper_with_first_chunk: bool, + ) -> Result { + let mut tamper_next = tamper_with_first_chunk; + let mut restarts = 0usize; + + 'snapshot_attempt: loop { + let offer_response = target_app.offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(snapshot.clone()), + app_hash: snapshot.hash.clone(), + })?; + assert_eq!( + offer_response.result, + i32::from(response_offer_snapshot::Result::Accept), + "target must accept the offered snapshot" + ); + + let mut chunk_queue: VecDeque> = VecDeque::from([snapshot.hash.clone()]); + + while let Some(chunk_id) = chunk_queue.pop_front() { + let chunk = source_app + .load_snapshot_chunk(proto::RequestLoadSnapshotChunk { + height: snapshot.height, + version: snapshot.version, + chunk_id: chunk_id.clone(), + })? + .chunk; + + if tamper_next { + tamper_next = false; + let mut tampered = chunk.clone(); + let last = tampered.len() - 1; + tampered[last] ^= 0xff; + + let response = + target_app.apply_snapshot_chunk(proto::RequestApplySnapshotChunk { + chunk_id: chunk_id.clone(), + chunk: tampered, + sender: "malicious-peer".to_string(), + })?; + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::RetrySnapshot), + "a tampered chunk must be answered with a snapshot restart, not an error" + ); + assert!(response.refetch_chunks.is_empty()); + assert_eq!(response.reject_senders, vec!["malicious-peer".to_string()]); + assert!( + target_app + .snapshot_fetching_session + .read() + .unwrap() + .is_some(), + "the session stays in place until the re-offer replaces it" + ); + restarts += 1; + continue 'snapshot_attempt; + } + + let response = + target_app.apply_snapshot_chunk(proto::RequestApplySnapshotChunk { + chunk_id, + chunk, + sender: "honest-peer".to_string(), + })?; + + match response.result { + result + if result == i32::from(response_apply_snapshot_chunk::Result::Accept) => + { + chunk_queue.extend(response.next_chunks); + } + result + if result + == i32::from( + response_apply_snapshot_chunk::Result::CompleteSnapshot, + ) => + { + assert!( + chunk_queue.is_empty(), + "transfer completed with chunks still queued" + ); + return Ok(SnapshotSyncOutcome::Completed); + } + result + if result + == i32::from(response_apply_snapshot_chunk::Result::RejectSnapshot) => + { + // The target restored the snapshot, found it unusable and wiped + // itself back to a clean slate. Tenderdash would try the next + // snapshot; there is nothing more for this driver to do. + assert!( + target_app + .snapshot_fetching_session + .read() + .unwrap() + .is_none(), + "a rejected snapshot must not leave a session open" + ); + return Ok(SnapshotSyncOutcome::Rejected); + } + result + if result + == i32::from(response_apply_snapshot_chunk::Result::RetrySnapshot) => + { + restarts += 1; + assert!(restarts <= 2, "too many snapshot restarts"); + continue 'snapshot_attempt; + } + other => panic!("unexpected apply_snapshot_chunk result {}", other), + } + } + + panic!("chunk transfer ran out of chunks without completing"); + } + } + + struct SourceChain<'a> { + source_app: FullAbciApplication<'a, MockCoreRPCLike>, + proposers: Vec, + validator_quorums: BTreeMap, + snapshot: proto::Snapshot, + } + + /// Runs the source chain past several checkpoints and picks its newest offered + /// snapshot. + async fn run_source_chain<'a>( + source_platform: &'a mut drive_abci::test::helpers::setup::TempPlatform, + config: &PlatformConfig, + ) -> SourceChain<'a> { + let ChainExecutionOutcome { + abci_app: source_app, + proposers, + validator_quorums, + .. + } = run_chain_for_strategy( + source_platform, + 15, + state_sync_network_strategy(), + config.clone(), + 15, + &mut None, + &mut None, + ) + .await; + + let snapshots = source_app + .list_snapshots(Default::default()) + .expect("source should list snapshots") + .snapshots; + assert!( + !snapshots.is_empty(), + "the source chain must have produced at least one restorable snapshot" + ); + let snapshot = snapshots + .iter() + .max_by_key(|snapshot| snapshot.height) + .expect("at least one snapshot") + .clone(); + + SourceChain { + source_app, + proposers: proposers + .iter() + .map(|proposer| proposer.masternode.clone()) + .collect(), + validator_quorums, + snapshot, + } + } + + /// End to end: run a source chain past several checkpoints, serve its newest + /// snapshot, restore it chunk by chunk on a fresh target (with one tampered chunk + /// along the way to prove refetch/restart recovery), reconstruct the target + /// platform state, and verify the target matches the source checkpoint exactly. + #[tokio::test] + #[ignore = "needs the grovedb sum-tree restore fix (dashpay/grovedb#840), which reaches \ + this workspace with the GroveDB 6.0.0 bump in #4635; un-ignore at that re-pin"] + async fn run_state_sync_between_two_platforms() { + let config = state_sync_platform_config(); + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let source = run_source_chain(&mut source_platform, &config).await; + let snapshot = &source.snapshot; + + // The platform state the source had at exactly the snapshot height + let source_platform_state = source + .source_app + .platform + .checkpoint_platform_states + .load() + .get(&snapshot.height) + .expect("source must cache the platform state of its checkpoint") + .clone(); + + // A fresh target node, knowing nothing but Core RPC + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + source.proposers.clone(), + &source.validator_quorums, + ); + let target_app = FullAbciApplication::new(&target_platform); + + assert_eq!( + sync_snapshot(&source.source_app, &target_app, snapshot, true) + .expect("state sync must not error"), + SnapshotSyncOutcome::Completed, + "state sync must complete" + ); + + let platform_version = PlatformVersion::latest(); + let grove_version = &platform_version.drive.grove_version; + + // Grove roots agree between source checkpoint and target + let target_root_hash = target_platform + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .expect("target root hash"); + assert_eq!(target_root_hash.to_vec(), snapshot.hash); + + // The restored grovedb is internally consistent + let verification_issues = target_platform + .drive + .grove + .verify_grovedb(None, true, false, grove_version) + .expect("expected to verify grovedb"); + assert!( + verification_issues.is_empty(), + "restored grovedb must verify cleanly: {:?}", + verification_issues + ); + + // The reconstructed platform state matches the source's state at the snapshot + // height, except for the fields that are not replicated (block signature and + // block id hash restore as zeroes) + let target_state = target_platform.state.load(); + assert_eq!( + target_state.current_protocol_version_in_consensus(), + source_platform_state.current_protocol_version_in_consensus() + ); + assert_eq!( + target_state.next_epoch_protocol_version(), + source_platform_state.next_epoch_protocol_version() + ); + assert_eq!( + target_state.last_committed_block_height(), + snapshot.height, + "target must be at the snapshot height" + ); + assert_eq!( + target_platform + .committed_block_height_guard + .load(std::sync::atomic::Ordering::Relaxed), + snapshot.height, + "a completed restore must open the query height gate at the snapshot height" + ); + assert_eq!( + target_state.last_committed_block_app_hash(), + source_platform_state.last_committed_block_app_hash() + ); + assert_eq!( + target_state.current_validator_set_quorum_hash(), + source_platform_state.current_validator_set_quorum_hash() + ); + assert_eq!( + target_state.next_validator_set_quorum_hash(), + source_platform_state.next_validator_set_quorum_hash() + ); + assert_eq!( + target_state.validator_sets().keys().collect::>(), + source_platform_state + .validator_sets() + .keys() + .collect::>(), + "validator set order must be restored from the recorded quorum positions" + ); + assert_eq!( + target_state.validator_sets(), + source_platform_state.validator_sets(), + "validator sets must match" + ); + assert_eq!( + target_state.full_masternode_list(), + source_platform_state.full_masternode_list() + ); + assert_eq!( + target_state.hpmn_masternode_list(), + source_platform_state.hpmn_masternode_list() + ); + assert_eq!( + target_state.previous_fee_versions(), + source_platform_state.previous_fee_versions(), + "fee versions of previous epochs must be restored faithfully" + ); + + // The target's info handler must pass its own app-hash consistency check and + // report the snapshot height and hash to Tenderdash's post-sync verifyApp + let info = target_app + .info(proto::RequestInfo { + version: tenderdash_abci::proto::meta::TENDERDASH_VERSION.to_string(), + block_version: 0, + p2p_version: 0, + abci_version: tenderdash_abci::proto::meta::ABCI_VERSION.to_string(), + }) + .expect("target info handler must succeed"); + assert_eq!(info.last_block_height as u64, snapshot.height); + assert_eq!(info.last_block_app_hash, snapshot.hash); + } + + /// Exercises the platform state reconstruction end to end without going through + /// the grovedb chunk restore: the source chain's + /// own grovedb IS a faithfully "restored" snapshot of itself, so reconstructing + /// on it must (a) not change the grovedb root hash — reconstruction rebuilds the + /// Core-derived state in memory and writes nothing to the replicated tree — and + /// (b) reproduce the source's in-memory platform state from the reduced platform + /// state alone. + #[tokio::test] + async fn platform_state_reconstruction_is_idempotent_and_matches_source_state() { + let config = state_sync_platform_config(); + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let source = run_source_chain(&mut source_platform, &config).await; + let platform = source.source_app.platform; + + let platform_version = PlatformVersion::latest(); + let grove_version = &platform_version.drive.grove_version; + + let original_state = platform.state.load().clone(); + let tip_app_hash = platform + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .expect("source root hash"); + assert_eq!( + original_state.last_committed_block_app_hash(), + Some(tip_app_hash), + "sanity: chain tip state matches grove root" + ); + + // The run_chain mocks already answer the from-scratch masternode/quorum + // requests reconstruction makes, exactly as they did for the chain itself. + platform + .reconstruct_platform_state(&tip_app_hash, platform_version) + .expect("platform state reconstruction must succeed"); + + // (a) reconstruction wrote nothing to the replicated tree + let root_hash_after = platform + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .expect("source root hash after reconstruction"); + assert_eq!( + root_hash_after, tip_app_hash, + "reconstruction must not change the grovedb root hash" + ); + + // (b) the reconstructed state matches the original, except the fields the + // reduced state cannot carry (block signature / block id hash) + let reconstructed_state = platform.state.load(); + assert_eq!( + reconstructed_state.current_protocol_version_in_consensus(), + original_state.current_protocol_version_in_consensus() + ); + assert_eq!( + reconstructed_state.next_epoch_protocol_version(), + original_state.next_epoch_protocol_version() + ); + assert_eq!( + reconstructed_state.last_committed_block_height(), + original_state.last_committed_block_height() + ); + assert_eq!( + reconstructed_state.last_committed_block_app_hash(), + original_state.last_committed_block_app_hash() + ); + assert_eq!( + reconstructed_state.last_committed_core_height(), + original_state.last_committed_core_height() + ); + assert_eq!( + reconstructed_state.current_validator_set_quorum_hash(), + original_state.current_validator_set_quorum_hash() + ); + assert_eq!( + reconstructed_state.next_validator_set_quorum_hash(), + original_state.next_validator_set_quorum_hash() + ); + assert_eq!( + reconstructed_state + .validator_sets() + .keys() + .collect::>(), + original_state.validator_sets().keys().collect::>(), + "validator set order must be restored from the recorded quorum positions" + ); + assert_eq!( + reconstructed_state.validator_sets(), + original_state.validator_sets() + ); + assert_eq!( + reconstructed_state.full_masternode_list(), + original_state.full_masternode_list() + ); + assert_eq!( + reconstructed_state.hpmn_masternode_list(), + original_state.hpmn_masternode_list() + ); + assert_eq!( + reconstructed_state.previous_fee_versions(), + original_state.previous_fee_versions() + ); + + // The info handler accepts the reconstructed state (it panics on an app-hash + // mismatch between the in-memory state and the grove root) + let info = source + .source_app + .info(proto::RequestInfo { + version: tenderdash_abci::proto::meta::TENDERDASH_VERSION.to_string(), + block_version: 0, + p2p_version: 0, + abci_version: tenderdash_abci::proto::meta::ABCI_VERSION.to_string(), + }) + .expect("info handler must accept the reconstructed state"); + assert_eq!( + info.last_block_height as u64, + original_state.last_committed_block_height() + ); + assert_eq!(info.last_block_app_hash, tip_app_hash.to_vec()); + } + + /// Checkpoints are created under the operator-configured `CHECKPOINTS_PATH`, so + /// startup has to read them back from the SAME place. When the reload path was + /// hard-coded to `/checkpoints`, a node configured with a custom path came + /// back from a restart with an empty registry: it stopped advertising the snapshots it + /// had retained, and could never prune the directories it had written. + #[tokio::test] + async fn checkpoints_in_a_custom_path_are_reloaded_after_a_restart() { + let checkpoints_dir = tempfile::tempdir().expect("should create a checkpoints dir"); + let mut config = state_sync_platform_config(); + config.abci.state_sync.checkpoints_path = Some(checkpoints_dir.path().to_path_buf()); + + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + + let (heights_before, db_path) = { + let outcome = run_chain_for_strategy( + &mut platform.platform, + 15, + state_sync_network_strategy(), + config.clone(), + 15, + &mut None, + &mut None, + ) + .await; + + let source = outcome.abci_app.platform; + let heights: Vec = source.drive.checkpoints.load().keys().copied().collect(); + (heights, source.config.db_path.clone()) + }; + + assert!( + !heights_before.is_empty(), + "the chain must have created checkpoints" + ); + assert!( + checkpoints_dir + .path() + .join(heights_before[0].to_string()) + .is_dir(), + "checkpoints must be written under the configured path" + ); + assert!( + !db_path.join("checkpoints").exists(), + "nothing must be written to the default path when one is configured" + ); + + let TempPlatform { + platform: original, + tempdir, + .. + } = platform; + drop(original); + let restarted = TempPlatform::open_with_tempdir(tempdir, config.clone()); + + let heights_after: Vec = restarted.drive.checkpoints.load().keys().copied().collect(); + assert_eq!( + heights_after, heights_before, + "a restart must reload the checkpoints from the configured path" + ); + + let app = FullAbciApplication::new(&restarted); + assert!( + !app.list_snapshots(Default::default()) + .expect("should list snapshots") + .snapshots + .is_empty(), + "a restarted node must keep advertising the snapshots it retained" + ); + } + + /// A snapshot from a chain that never wrote the reduced platform state (pre-v15) + /// is not offered by the source, and a target driven at it anyway refuses to + /// restore it. + #[tokio::test] + async fn pre_v15_snapshot_is_not_served_and_cannot_be_restored() { + let config = state_sync_platform_config(); + + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .with_initial_protocol_version(14) + .build_with_mock_rpc(); + + let ChainExecutionOutcome { + abci_app: source_app, + proposers, + validator_quorums, + .. + } = run_chain_for_strategy( + &mut source_platform, + 15, + state_sync_network_strategy(), + config.clone(), + 15, + &mut None, + &mut None, + ) + .await; + + // The v14 chain created checkpoints, but none carries the reduced platform + // state, so none may be offered. + assert!( + !source_app.platform.drive.checkpoints.load().is_empty(), + "the source must have created checkpoints" + ); + let snapshots = source_app + .list_snapshots(Default::default()) + .expect("source should list snapshots") + .snapshots; + assert!( + snapshots.is_empty(), + "pre-v15 checkpoints are unrestorable and must not be offered" + ); + + // Offered honestly — with its real, pre-v15 protocol version — such a snapshot is + // refused before anything is wiped. + let honest_pre_v15_offer = proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1, + version: 1, + hash: vec![7u8; 32], + metadata: encode_snapshot_metadata(14), + }), + app_hash: vec![7u8; 32], + }; + + // Even if a peer maliciously offers such a snapshot — lying in the metadata that + // it is restorable — the target must refuse to restore it: the chunk transfer + // completes, but the reconstruction step finds no reduced platform state. + let (height, checkpoint) = { + let checkpoints = source_app.platform.drive.checkpoints.load(); + let (height, info) = checkpoints + .last_key_value() + .expect("at least one checkpoint"); + (*height, std::sync::Arc::clone(&info.checkpoint)) + }; + let platform_version = PlatformVersion::latest(); + let checkpoint_root = checkpoint + .grove_db + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("checkpoint root hash"); + let forged_snapshot = proto::Snapshot { + height, + version: 1, + hash: checkpoint_root.to_vec(), + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), + }; + + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + proposers + .iter() + .map(|proposer| proposer.masternode.clone()) + .collect(), + &validator_quorums, + ); + let target_app = FullAbciApplication::new(&target_platform); + + let honest_offer_response = target_app + .offer_snapshot(honest_pre_v15_offer) + .expect("an honestly-labelled pre-v15 offer is answered, not errored"); + assert_eq!( + honest_offer_response.result, + i32::from(response_offer_snapshot::Result::Reject), + "a snapshot that declares a pre-v15 protocol version must be refused up front" + ); + + let outcome = sync_snapshot(&source_app, &target_app, &forged_snapshot, false) + .expect("a refused snapshot is answered, not errored"); + assert_eq!( + outcome, + SnapshotSyncOutcome::Rejected, + "a snapshot without the reduced platform state must be refused" + ); + + // The target holds no usable platform state: it never advanced past genesis + assert_eq!( + target_platform.state.load().last_committed_block_height(), + 0 + ); + // ...and it did not keep the state it could not use: the refusal wipes back to a + // clean slate so Tenderdash can offer another snapshot or fall back to block sync. + assert_ne!( + target_platform + .drive + .grove + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("target root hash") + .to_vec(), + forged_snapshot.hash, + "a refused snapshot must not be left on disk" + ); + } +} diff --git a/packages/rs-drive/src/drive/mod.rs b/packages/rs-drive/src/drive/mod.rs index 98eeafa0c24..f2e20d43327 100644 --- a/packages/rs-drive/src/drive/mod.rs +++ b/packages/rs-drive/src/drive/mod.rs @@ -11,6 +11,10 @@ use crate::config::DriveConfig; use arc_swap::ArcSwap; #[cfg(feature = "server")] use dpp::prelude::{BlockHeight, TimestampMillis}; +#[cfg(feature = "server")] +use dpp::util::deserializer::ProtocolVersion; +#[cfg(feature = "server")] +use dpp::version::PlatformVersion; #[cfg(any(feature = "server", feature = "verify"))] use grovedb::GroveDb; use std::fmt; @@ -106,6 +110,47 @@ impl Checkpoint { } } + /// Returns true if this checkpoint contains the reduced platform state + /// (`Misc/reduced_saved_state`), which a state-syncing node needs to reconstruct the + /// platform state. Checkpoints taken before the protocol version that introduced the + /// reduced state lack the key and cannot be offered as state sync snapshots. + pub fn has_reduced_platform_state( + &self, + grove_version: &grovedb_version::version::GroveVersion, + ) -> Result { + self.grove_db + .get_raw_optional( + (&crate::drive::system::misc_path()).into(), + crate::drive::platform_state::REDUCED_PLATFORM_STATE_KEY, + None, + grove_version, + ) + .unwrap() + .map(|maybe_element| maybe_element.is_some()) + .map_err(Error::from) + } + + /// The Platform protocol version the chain was running at when this checkpoint was + /// taken, as recorded in the checkpoint's own aux storage. + /// + /// State sync must serve and consume a snapshot under the version table the snapshot + /// was PRODUCED with — the consuming node is typically a fresh node whose in-memory + /// version is still the initial one — so this is the authoritative source for it. + pub fn current_protocol_version(&self) -> Result, Error> { + Drive::fetch_current_protocol_version_with_grovedb(&self.grove_db, None) + } + + /// The Platform version this checkpoint must be read under, or `None` when the + /// checkpoint records no protocol version or one this binary does not know. + /// + /// Both are reasons not to serve the checkpoint as a state sync snapshot rather than + /// errors: an unknown version is simply a newer node's checkpoint. + pub fn platform_version(&self) -> Result, Error> { + Ok(self + .current_protocol_version()? + .and_then(|protocol_version| PlatformVersion::get(protocol_version).ok())) + } + /// Marks this checkpoint for deletion when it is dropped. pub fn mark_for_deletion(&self) { self.marked_for_deletion diff --git a/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs new file mode 100644 index 00000000000..2045c905235 --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs @@ -0,0 +1,33 @@ +mod v0; + +use crate::drive::Drive; +use crate::error::drive::DriveError; +use crate::error::Error; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; + +impl Drive { + /// Fetch the reduced platform state from the replicated grovedb state (Misc tree). + /// + /// Returns `Ok(None)` when the key is absent (for example before the protocol + /// version that introduced the reduced state activated). + pub fn fetch_reduced_platform_state_bytes( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result>, Error> { + match platform_version + .drive + .methods + .platform_state + .fetch_reduced_platform_state_bytes + { + 0 => self.fetch_reduced_platform_state_bytes_v0(transaction, platform_version), + version => Err(Error::Drive(DriveError::UnknownVersionMismatch { + method: "fetch_reduced_platform_state_bytes".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs new file mode 100644 index 00000000000..7df67f7c084 --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs @@ -0,0 +1,24 @@ +use crate::drive::platform_state::REDUCED_PLATFORM_STATE_KEY; +use crate::drive::system::misc_path; +use crate::drive::Drive; +use crate::error::Error; +use crate::util::grove_operations::DirectQueryType; +use grovedb::TransactionArg; +use platform_version::version::PlatformVersion; + +impl Drive { + pub(super) fn fetch_reduced_platform_state_bytes_v0( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result>, Error> { + self.grove_get_raw_optional_item( + (&misc_path()).into(), + REDUCED_PLATFORM_STATE_KEY, + DirectQueryType::StatefulDirectQuery, + transaction, + &mut vec![], + &platform_version.drive, + ) + } +} diff --git a/packages/rs-drive/src/drive/platform_state/mod.rs b/packages/rs-drive/src/drive/platform_state/mod.rs index d6a0ce16c49..9b100fc239d 100644 --- a/packages/rs-drive/src/drive/platform_state/mod.rs +++ b/packages/rs-drive/src/drive/platform_state/mod.rs @@ -1,4 +1,64 @@ mod fetch_platform_state_bytes; +mod fetch_reduced_platform_state_bytes; mod store_platform_state_bytes; +mod store_reduced_platform_state_bytes; const PLATFORM_STATE_KEY: &[u8; 11] = b"saved_state"; +pub(crate) const REDUCED_PLATFORM_STATE_KEY: &[u8; 19] = b"reduced_saved_state"; + +#[cfg(test)] +mod tests { + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + use platform_version::version::PlatformVersion; + + #[test] + fn should_return_none_when_reduced_platform_state_is_absent() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + let fetched = drive + .fetch_reduced_platform_state_bytes(None, platform_version) + .expect("fetching an absent reduced platform state should not error"); + + assert_eq!(fetched, None); + } + + #[test] + fn should_roundtrip_reduced_platform_state_bytes() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + let state_bytes = vec![1u8, 2, 3, 4, 5]; + + drive + .store_reduced_platform_state_bytes(&state_bytes, None, platform_version) + .expect("should store reduced platform state"); + + let fetched = drive + .fetch_reduced_platform_state_bytes(None, platform_version) + .expect("should fetch reduced platform state"); + + assert_eq!(fetched, Some(state_bytes)); + } + + #[test] + fn should_overwrite_reduced_platform_state_bytes() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + drive + .store_reduced_platform_state_bytes(&[1u8, 2, 3], None, platform_version) + .expect("should store reduced platform state"); + + let updated_bytes = vec![9u8, 8, 7]; + drive + .store_reduced_platform_state_bytes(&updated_bytes, None, platform_version) + .expect("should overwrite reduced platform state"); + + let fetched = drive + .fetch_reduced_platform_state_bytes(None, platform_version) + .expect("should fetch reduced platform state"); + + assert_eq!(fetched, Some(updated_bytes)); + } +} diff --git a/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs new file mode 100644 index 00000000000..0346e197218 --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs @@ -0,0 +1,35 @@ +mod v0; + +use crate::drive::Drive; +use crate::error::drive::DriveError; +use crate::error::Error; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; + +impl Drive { + /// Store the reduced platform state in the replicated grovedb state (Misc tree) + pub fn store_reduced_platform_state_bytes( + &self, + state_bytes: &[u8], + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match platform_version + .drive + .methods + .platform_state + .store_reduced_platform_state_bytes + { + 0 => self.store_reduced_platform_state_bytes_v0( + state_bytes, + transaction, + platform_version, + ), + version => Err(Error::Drive(DriveError::UnknownVersionMismatch { + method: "store_reduced_platform_state_bytes".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs new file mode 100644 index 00000000000..55d61b2e08a --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs @@ -0,0 +1,28 @@ +use crate::drive::platform_state::REDUCED_PLATFORM_STATE_KEY; +use crate::drive::system::misc_path; +use crate::drive::Drive; +use crate::error::Error; +use grovedb::{Element, TransactionArg}; +use platform_version::version::PlatformVersion; + +impl Drive { + pub(super) fn store_reduced_platform_state_bytes_v0( + &self, + reduced_state_bytes: &[u8], + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + self.grove + .insert( + &misc_path(), + REDUCED_PLATFORM_STATE_KEY, + Element::Item(reduced_state_bytes.to_vec(), None), + None, + transaction, + &platform_version.drive.grove_version, + ) + .unwrap() + .map_err(Error::from)?; + Ok(()) + } +} diff --git a/packages/rs-drive/src/error/drive.rs b/packages/rs-drive/src/error/drive.rs index 3412c9df1aa..a64d5c94adb 100644 --- a/packages/rs-drive/src/error/drive.rs +++ b/packages/rs-drive/src/error/drive.rs @@ -218,4 +218,8 @@ pub enum DriveError { /// Checkpoint not found for specified block height #[error("checkpoint not found for block height: {0}")] CheckpointNotFound(u64), + + /// Snapshot error + #[error("snapshot error: {0}")] + Snapshot(String), } diff --git a/packages/rs-drive/src/open/load_current_checkpoints.rs b/packages/rs-drive/src/open/load_current_checkpoints.rs index 99792a5f059..c94a0b794fc 100644 --- a/packages/rs-drive/src/open/load_current_checkpoints.rs +++ b/packages/rs-drive/src/open/load_current_checkpoints.rs @@ -11,17 +11,24 @@ use crate::error::Error; /// Loads existing checkpoints from the checkpoints directory. /// -/// This function scans the `/checkpoints/` directory for existing checkpoint +/// This function scans the given checkpoints directory for existing checkpoint /// subdirectories (named by block height), opens each one as a GroveDb, and returns /// an ArcSwap containing the loaded checkpoints. /// +/// The directory is passed in rather than derived, because checkpoints may be configured +/// to live outside the database directory (`CHECKPOINTS_PATH`). Deriving it here would +/// leave a node restarted with a custom path holding an empty registry: it would stop +/// advertising its retained snapshots and could never prune the directories it wrote. +/// /// # Arguments -/// * `db_path` - The path to the database directory (parent of the checkpoints directory) +/// * `checkpoints_dir` - The directory checkpoints are written to /// /// # Returns /// * An `ArcSwap` containing a `BTreeMap` of checkpoints keyed by block height -pub fn load_current_checkpoints>(db_path: P) -> Result { - let checkpoints_dir = db_path.as_ref().join("checkpoints"); +pub fn load_current_checkpoints>( + checkpoints_dir: P, +) -> Result { + let checkpoints_dir = checkpoints_dir.as_ref(); let mut checkpoints = BTreeMap::new(); @@ -31,7 +38,7 @@ pub fn load_current_checkpoints>(db_path: P) -> Result entries, Err(_) => return Ok(ArcSwap::from_pointee(checkpoints)), }; diff --git a/packages/rs-drive/src/open/mod.rs b/packages/rs-drive/src/open/mod.rs index e51fb1aab19..ede4e2f00cd 100644 --- a/packages/rs-drive/src/open/mod.rs +++ b/packages/rs-drive/src/open/mod.rs @@ -30,6 +30,27 @@ impl Drive { pub fn open>( path: P, config: Option, + ) -> Result<(Self, Option<&'static PlatformVersion>), Error> { + let checkpoints_path = path.as_ref().join("checkpoints"); + Self::open_with_checkpoints_path(path, config, checkpoints_path) + } + + /// Opens GroveDB database, loading the checkpoint registry from an explicit directory. + /// + /// Checkpoints may be configured to live outside the database directory + /// (`CHECKPOINTS_PATH`). Whoever knows that configuration must pass the same directory + /// checkpoint creation writes to, otherwise the registry comes up empty after a + /// restart and the checkpoints on disk are neither advertised nor prunable. + /// + /// # Arguments + /// + /// * `path` - The path to the GroveDB. + /// * `config` - An `Option` which contains `DriveConfig`. If not specified, default configuration is used. + /// * `checkpoints_path` - The directory checkpoints are written to. + pub fn open_with_checkpoints_path, Q: AsRef>( + path: P, + config: Option, + checkpoints_path: Q, ) -> Result<(Self, Option<&'static PlatformVersion>), Error> { let config = config.unwrap_or_default(); let db_path = path.as_ref(); @@ -52,8 +73,8 @@ impl Drive { }) .transpose()?; - // Load existing checkpoints from the checkpoints directory - let checkpoints = load_current_checkpoints(db_path)?; + // Load existing checkpoints from the configured checkpoints directory + let checkpoints = load_current_checkpoints(checkpoints_path)?; let drive = Drive { grove, diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs index a1bf5fdd754..cfd94cf9969 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs @@ -2,6 +2,7 @@ use versioned_feature_core::{FeatureVersion, OptionalFeatureVersion}; pub mod v1; pub mod v10; +pub mod v11; pub mod v2; pub mod v3; pub mod v4; @@ -36,6 +37,8 @@ pub struct DriveAbciMethodVersions { pub struct DriveAbciPlatformStateStorageMethodVersions { pub fetch_platform_state: FeatureVersion, pub store_platform_state: FeatureVersion, + pub fetch_reduced_platform_state: FeatureVersion, + pub store_reduced_platform_state: FeatureVersion, } #[derive(Clone, Copy, Debug, Default)] diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs index 9798d693037..b03335c73d7 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs @@ -131,5 +131,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V1: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs index 38be834a186..a88ace56cab 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs @@ -140,5 +140,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V10: DriveAbciMethodVersions = DriveAbciMet platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs new file mode 100644 index 00000000000..8ea89d7169d --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs @@ -0,0 +1,146 @@ +use crate::version::drive_abci_versions::drive_abci_method_versions::{ + DriveAbciBlockEndMethodVersions, DriveAbciBlockFeeProcessingMethodVersions, + DriveAbciBlockStartMethodVersions, DriveAbciCoreBasedUpdatesMethodVersions, + DriveAbciCoreChainLockMethodVersionsAndConstants, DriveAbciCoreInstantSendLockMethodVersions, + DriveAbciEngineMethodVersions, DriveAbciEpochMethodVersions, + DriveAbciFeePoolInwardsDistributionMethodVersions, + DriveAbciFeePoolOutwardsDistributionMethodVersions, + DriveAbciIdentityCreditWithdrawalMethodVersions, DriveAbciInitializationMethodVersions, + DriveAbciMasternodeIdentitiesUpdatesMethodVersions, DriveAbciMethodVersions, + DriveAbciPlatformStateStorageMethodVersions, DriveAbciProtocolUpgradeMethodVersions, + DriveAbciStateTransitionProcessingMethodVersions, DriveAbciTokensProcessingMethodVersions, + DriveAbciVotingMethodVersions, +}; + +/// Drive ABCI method versions 11. Introduced in protocol v15 for state sync: +/// `run_block_proposal` 0 -> 1 (the reduced platform state is written into the replicated +/// state each block, and `validator_set_update` moves above the root-hash computation so the +/// stored reduced state is sufficient to reconstruct the post-rotation state), and +/// `consensus_params_update` 1 -> 2 (emits evidence params when crossing to v15). +/// Everything else matches `DRIVE_ABCI_METHOD_VERSIONS_V10`. +pub const DRIVE_ABCI_METHOD_VERSIONS_V11: DriveAbciMethodVersions = DriveAbciMethodVersions { + engine: DriveAbciEngineMethodVersions { + init_chain: 0, + check_tx: 0, + run_block_proposal: 1, + finalize_block_proposal: 0, + consensus_params_update: 2, + }, + initialization: DriveAbciInitializationMethodVersions { + initial_core_height_and_time: 0, + create_genesis_state: 1, + }, + core_based_updates: DriveAbciCoreBasedUpdatesMethodVersions { + update_core_info: 0, + update_masternode_list: 0, + update_quorum_info: 0, + masternode_updates: DriveAbciMasternodeIdentitiesUpdatesMethodVersions { + get_voter_identity_key: 0, + get_operator_identity_keys: 0, + get_owner_identity_withdrawal_key: 0, + get_owner_identity_owner_key: 0, + get_voter_identifier_from_masternode_list_item: 0, + get_operator_identifier_from_masternode_list_item: 0, + create_operator_identity: 0, + create_owner_identity: 1, + create_voter_identity: 0, + disable_identity_keys: 0, + update_masternode_identities: 0, + update_operator_identity: 0, + update_owner_withdrawal_address: 1, + update_voter_identity: 0, + }, + }, + protocol_upgrade: DriveAbciProtocolUpgradeMethodVersions { + check_for_desired_protocol_upgrade: 1, + upgrade_protocol_version_on_epoch_change: 0, + perform_events_on_first_block_of_protocol_change: Some(1), + protocol_version_upgrade_percentage_needed: 67, + }, + block_fee_processing: DriveAbciBlockFeeProcessingMethodVersions { + add_process_epoch_change_operations: 0, + process_block_fees_and_validate_sum_trees: 1, + }, + tokens_processing: DriveAbciTokensProcessingMethodVersions { + validate_token_aggregated_balance: 0, + }, + core_chain_lock: DriveAbciCoreChainLockMethodVersionsAndConstants { + choose_quorum: 0, + verify_chain_lock: 0, + verify_chain_lock_locally: 0, + verify_chain_lock_through_core: 0, + make_sure_core_is_synced_to_chain_lock: 0, + recent_block_count_amount: 2, + }, + core_instant_send_lock: DriveAbciCoreInstantSendLockMethodVersions { + verify_recent_signature_locally: 0, + }, + fee_pool_inwards_distribution: DriveAbciFeePoolInwardsDistributionMethodVersions { + add_distribute_block_fees_into_pools_operations: 0, + add_distribute_storage_fee_to_epochs_operations: 0, + }, + fee_pool_outwards_distribution: DriveAbciFeePoolOutwardsDistributionMethodVersions { + add_distribute_fees_from_oldest_unpaid_epoch_pool_to_proposers_operations: 1, + add_epoch_pool_to_proposers_payout_operations: 0, + find_oldest_epoch_needing_payment: 0, + fetch_reward_shares_list_for_masternode: 0, + }, + withdrawals: DriveAbciIdentityCreditWithdrawalMethodVersions { + build_untied_withdrawal_transactions_from_documents: 0, + dequeue_and_build_unsigned_withdrawal_transactions: 0, + fetch_transactions_block_inclusion_status: 0, + pool_withdrawals_into_transactions_queue: 1, + update_broadcasted_withdrawal_statuses: 0, + rebroadcast_expired_withdrawal_documents: 1, + append_signatures_and_broadcast_withdrawal_transactions: 0, + cleanup_expired_locks_of_withdrawal_amounts: 1, // changed in v14: also prunes expired entries of the credit inflows sum tree + record_credit_inflows_for_withdrawals: Some(0), // new in v14: the block's credit mints recorded as an inflow for the net daily withdrawal limit + record_total_credits_history_for_withdrawals: Some(0), // changed in v14: per-block total credits history for the day-lagged daily withdrawal limit + }, + voting: DriveAbciVotingMethodVersions { + keep_record_of_finished_contested_resource_vote_poll: 0, + clean_up_after_vote_poll_end: 0, + clean_up_after_contested_resources_vote_poll_end: 1, + check_for_ended_vote_polls: 0, + tally_votes_for_contested_document_resource_vote_poll: 0, + award_document_to_winner: 0, + delay_vote_poll: 0, + run_dao_platform_events: 0, + remove_votes_for_removed_masternodes: 0, + }, + state_transition_processing: DriveAbciStateTransitionProcessingMethodVersions { + execute_event: 0, + process_raw_state_transitions: 0, + // unchanged from V9: v1 since v13 (records the balance effects of paid-INVALID / + // unsuccessful-paid transitions) + process_validation_result: 1, + decode_raw_state_transitions: 0, + validate_fees_of_event: 0, + store_address_balances_to_recent_block_storage: Some(0), + cleanup_recent_block_storage_address_balances: Some(0), + // unchanged from V9: v1 since v13 (records shielded-spend transparent credits) + record_added_balance_outputs: 1, + }, + epoch: DriveAbciEpochMethodVersions { + gather_epoch_info: 0, + get_genesis_time: 0, + }, + block_start: DriveAbciBlockStartMethodVersions { + clear_drive_block_cache: 0, + }, + block_end: DriveAbciBlockEndMethodVersions { + update_state_cache: 0, + update_drive_cache: 0, + validator_set_update: 2, + should_checkpoint: Some(0), + update_checkpoints: Some(0), + record_shielded_pool_anchor: Some(0), + prune_shielded_pool_anchors: Some(0), + }, + platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { + fetch_platform_state: 0, + store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, + }, +}; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs index c3177e006f7..e781dbca981 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs @@ -132,5 +132,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V2: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs index 06fe75413e5..e9a0fb51daa 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs @@ -131,5 +131,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V3: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs index 843e71c6d40..b5c18a3c727 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs @@ -131,5 +131,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V4: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs index bed7af26bf2..84d1c011f38 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs @@ -135,5 +135,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V5: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs index df56f534d95..f696c3d7dba 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs @@ -133,5 +133,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V6: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs index da44e9b81a9..6a10f182858 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs @@ -142,5 +142,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V7: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs index d9a461cc62f..0629a7808c7 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs @@ -142,5 +142,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V8: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs index 434f216c461..f3d4d857fd0 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs @@ -160,5 +160,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V9: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs new file mode 100644 index 00000000000..f0158bc6ad9 --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs @@ -0,0 +1,14 @@ +pub mod v1; + +use versioned_feature_core::FeatureVersion; + +/// Versions for ABCI state sync (snapshot serving and consumption). +#[derive(Clone, Debug, Default)] +pub struct DriveAbciStateSyncVersions { + /// The grovedb state sync protocol version used for snapshots this node creates + /// and serves. Exactly one version exists (grovedb updates its replication + /// protocol in place and stays at version 1); snapshots offered by peers are + /// validated against the supported set in `drive-abci`'s snapshot module so any + /// future incompatible protocol change fails fast on both sides. + pub protocol_version: FeatureVersion, +} diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs new file mode 100644 index 00000000000..f5a3d13e884 --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs @@ -0,0 +1,6 @@ +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::DriveAbciStateSyncVersions; + +pub const DRIVE_ABCI_STATE_SYNC_VERSIONS_V1: DriveAbciStateSyncVersions = + DriveAbciStateSyncVersions { + protocol_version: 1, + }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs index 6df817b3dfd..9adde49dbe5 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs @@ -1,6 +1,7 @@ pub mod drive_abci_checkpoint_parameters; pub mod drive_abci_method_versions; pub mod drive_abci_query_versions; +pub mod drive_abci_state_sync_versions; pub mod drive_abci_structure_versions; pub mod drive_abci_validation_versions; pub mod drive_abci_withdrawal_constants; @@ -8,6 +9,7 @@ pub mod drive_abci_withdrawal_constants; use drive_abci_checkpoint_parameters::DriveAbciCheckpointParameters; use drive_abci_method_versions::DriveAbciMethodVersions; use drive_abci_query_versions::DriveAbciQueryVersions; +use drive_abci_state_sync_versions::DriveAbciStateSyncVersions; use drive_abci_structure_versions::DriveAbciStructureVersions; use drive_abci_validation_versions::DriveAbciValidationVersions; use drive_abci_withdrawal_constants::DriveAbciWithdrawalConstants; @@ -20,4 +22,5 @@ pub struct DriveAbciVersion { pub withdrawal_constants: DriveAbciWithdrawalConstants, pub query: DriveAbciQueryVersions, pub checkpoints: DriveAbciCheckpointParameters, + pub state_sync: DriveAbciStateSyncVersions, } diff --git a/packages/rs-platform-version/src/version/drive_versions/mod.rs b/packages/rs-platform-version/src/version/drive_versions/mod.rs index ca7c22c6e3f..c768c5a36a1 100644 --- a/packages/rs-platform-version/src/version/drive_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_versions/mod.rs @@ -78,6 +78,8 @@ pub struct DriveMethodVersions { pub struct DrivePlatformStateMethodVersions { pub fetch_platform_state_bytes: FeatureVersion, pub store_platform_state_bytes: FeatureVersion, + pub fetch_reduced_platform_state_bytes: FeatureVersion, + pub store_reduced_platform_state_bytes: FeatureVersion, } #[derive(Clone, Debug, Default)] diff --git a/packages/rs-platform-version/src/version/drive_versions/v1.rs b/packages/rs-platform-version/src/version/drive_versions/v1.rs index 6e87d8fe420..f7dc6e68748 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v1.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v1.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V1: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v2.rs b/packages/rs-platform-version/src/version/drive_versions/v2.rs index 0fe4f8f235e..02a4ab14d14 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v2.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v2.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V2: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v3.rs b/packages/rs-platform-version/src/version/drive_versions/v3.rs index a542fe99e85..13d5a29cc94 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v3.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v3.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V3: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v4.rs b/packages/rs-platform-version/src/version/drive_versions/v4.rs index 4481d8b90ac..d2c09c3a2fd 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v4.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v4.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V4: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v5.rs b/packages/rs-platform-version/src/version/drive_versions/v5.rs index bfbce3d74b1..6cd9ffdefe8 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v5.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v5.rs @@ -92,6 +92,8 @@ pub const DRIVE_VERSION_V5: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v6.rs b/packages/rs-platform-version/src/version/drive_versions/v6.rs index 304cbdb70c7..7b265daeafa 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v6.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v6.rs @@ -94,6 +94,8 @@ pub const DRIVE_VERSION_V6: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v7.rs b/packages/rs-platform-version/src/version/drive_versions/v7.rs index 05d8ad2d05e..3761c8fbd6d 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v7.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v7.rs @@ -92,6 +92,8 @@ pub const DRIVE_VERSION_V7: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v8.rs b/packages/rs-platform-version/src/version/drive_versions/v8.rs index 7f421173191..f48deed6d37 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v8.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v8.rs @@ -92,6 +92,8 @@ pub const DRIVE_VERSION_V8: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v9.rs b/packages/rs-platform-version/src/version/drive_versions/v9.rs index fade08c521d..a9cd31b2da7 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v9.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v9.rs @@ -106,6 +106,8 @@ pub const DRIVE_VERSION_V9: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/fee/mod.rs b/packages/rs-platform-version/src/version/fee/mod.rs index 8b603e2ddba..a196f3972ce 100644 --- a/packages/rs-platform-version/src/version/fee/mod.rs +++ b/packages/rs-platform-version/src/version/fee/mod.rs @@ -29,6 +29,12 @@ pub mod vote_resolution_fund_fees; pub type FeeVersionNumber = u32; +/// The fee schedules [`FeeVersion::get`] can resolve, indexed by `fee_version_number - 1`. +/// +/// Every `FeeVersion` needs a unique `fee_version_number` and an entry here at the index +/// that number implies, because only the number is persisted. +/// +/// BUG(#4647): incomplete. `FEE_VERSION2` is missing and reuses number 1; see its doc comment. pub const FEE_VERSIONS: &[FeeVersion] = &[FEE_VERSION1]; #[derive(Clone, Debug, Encode, Decode, Default, PartialEq, Eq)] @@ -116,3 +122,49 @@ impl From for FeeVersion { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::version::fee::v2::FEE_VERSION2; + + /// Every `FeeVersion` constant needs a distinct `fee_version_number` that resolves + /// back to it through `FEE_VERSIONS`, because only the number is persisted. Fails + /// today (`FEE_VERSION2` reuses number 1, see its doc comment); un-ignore with the fix. + #[test] + #[ignore = "known defect: FEE_VERSION2 reuses fee_version_number 1 and is absent from \ + FEE_VERSIONS; fixing it is protocol-visible - see the FEE_VERSION2 docs"] + fn fee_version_numbers_are_unique_and_resolvable() { + let all_fee_versions = [&FEE_VERSION1, &FEE_VERSION2]; + + for fee_version in all_fee_versions { + let resolved = FeeVersion::get(fee_version.fee_version_number).unwrap_or_else(|_| { + panic!( + "fee version number {} does not resolve through FEE_VERSIONS", + fee_version.fee_version_number + ) + }); + assert_eq!( + resolved, fee_version, + "FeeVersion::get({}) returned a DIFFERENT fee schedule than the constant \ + declaring that number. Every number-only round trip - a node restarting, a \ + node state-syncing - would substitute this wrong schedule.", + fee_version.fee_version_number + ); + } + + let mut numbers: Vec = all_fee_versions + .iter() + .map(|fee_version| fee_version.fee_version_number) + .collect(); + numbers.sort_unstable(); + let mut deduped = numbers.clone(); + deduped.dedup(); + assert_eq!( + deduped.len(), + numbers.len(), + "two FeeVersion constants share a fee_version_number: {:?}", + numbers + ); + } +} diff --git a/packages/rs-platform-version/src/version/fee/v2.rs b/packages/rs-platform-version/src/version/fee/v2.rs index fe82ac5534f..a690d8e864c 100644 --- a/packages/rs-platform-version/src/version/fee/v2.rs +++ b/packages/rs-platform-version/src/version/fee/v2.rs @@ -9,8 +9,16 @@ use crate::version::fee::vote_resolution_fund_fees::v1::VOTE_RESOLUTION_FUND_FEE use crate::version::fee::FeeVersion; /// Introduced in protocol version 9 (2.0) +/// +/// BUG(#4647): `fee_version_number` collides with `FEE_VERSION1` and this constant is missing +/// from `FEE_VERSIONS`, so `FeeVersion::get(1)` never resolves to it. The number is the +/// only thing persisted (`PlatformStateForSavingV1`, `ReducedPlatformStateV0`), so a +/// node that restarts or state-syncs rehydrates previous epochs' fees as `FEE_VERSION1`. +/// Latent only because the two share identical storage fees, which is all +/// `previous_fee_versions` is consulted for. Giving it number 2 is protocol-visible and +/// needs a versioned migration; see `fee_version_numbers_are_unique_and_resolvable`. pub const FEE_VERSION2: FeeVersion = FeeVersion { - fee_version_number: 1, + fee_version_number: 1, // BUG: must be 2, see the doc comment above uses_version_fee_multiplier_permille: Some(1000), //No action storage: FEE_STORAGE_VERSION1, signature: FEE_SIGNATURE_VERSION1, diff --git a/packages/rs-platform-version/src/version/mocks/v2_test.rs b/packages/rs-platform-version/src/version/mocks/v2_test.rs index 76579f3a0d7..838b110151d 100644 --- a/packages/rs-platform-version/src/version/mocks/v2_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v2_test.rs @@ -23,6 +23,7 @@ use crate::version::drive_abci_versions::drive_abci_query_versions::{ DriveAbciQueryShieldedVersions, DriveAbciQuerySystemVersions, DriveAbciQueryTokenVersions, DriveAbciQueryValidatorVersions, DriveAbciQueryVersions, DriveAbciQueryVotingVersions, }; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v1::DRIVE_ABCI_VALIDATION_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -128,6 +129,8 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { @@ -478,6 +481,7 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { }, }, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/mocks/v3_test.rs b/packages/rs-platform-version/src/version/mocks/v3_test.rs index e5cfd15b1bb..d6325c633c0 100644 --- a/packages/rs-platform-version/src/version/mocks/v3_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v3_test.rs @@ -29,6 +29,7 @@ use crate::version::drive_abci_versions::drive_abci_method_versions::{ DriveAbciVotingMethodVersions, }; use crate::version::drive_abci_versions::drive_abci_query_versions::v1::DRIVE_ABCI_QUERY_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v3::DRIVE_ABCI_VALIDATION_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -166,12 +167,15 @@ pub const TEST_PLATFORM_V3: PlatformVersion = PlatformVersion { platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }, validation_and_processing: DRIVE_ABCI_VALIDATION_VERSIONS_V3, withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V1, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/mod.rs b/packages/rs-platform-version/src/version/mod.rs index 1b1635efb42..ae5fd0887e1 100644 --- a/packages/rs-platform-version/src/version/mod.rs +++ b/packages/rs-platform-version/src/version/mod.rs @@ -1,6 +1,6 @@ mod protocol_version; -use crate::version::v14::PROTOCOL_VERSION_14; +use crate::version::v15::PROTOCOL_VERSION_15; pub use protocol_version::*; use std::ops::RangeInclusive; @@ -20,6 +20,7 @@ pub mod v11; pub mod v12; pub mod v13; pub mod v14; +pub mod v15; pub mod v2; pub mod v3; pub mod v4; @@ -33,5 +34,5 @@ pub type ProtocolVersion = u32; pub const ALL_VERSIONS: RangeInclusive = 1..=LATEST_VERSION; -pub const LATEST_VERSION: ProtocolVersion = PROTOCOL_VERSION_14; +pub const LATEST_VERSION: ProtocolVersion = PROTOCOL_VERSION_15; pub const INITIAL_PROTOCOL_VERSION: ProtocolVersion = 1; diff --git a/packages/rs-platform-version/src/version/protocol_version.rs b/packages/rs-platform-version/src/version/protocol_version.rs index 0eded570c10..2ba05cc8366 100644 --- a/packages/rs-platform-version/src/version/protocol_version.rs +++ b/packages/rs-platform-version/src/version/protocol_version.rs @@ -22,6 +22,7 @@ use crate::version::v11::PLATFORM_V11; use crate::version::v12::PLATFORM_V12; use crate::version::v13::PLATFORM_V13; use crate::version::v14::PLATFORM_V14; +use crate::version::v15::PLATFORM_V15; use crate::version::v2::PLATFORM_V2; use crate::version::v3::PLATFORM_V3; use crate::version::v4::PLATFORM_V4; @@ -61,6 +62,7 @@ pub const PLATFORM_VERSIONS: &[PlatformVersion] = &[ PLATFORM_V12, PLATFORM_V13, PLATFORM_V14, + PLATFORM_V15, ]; #[cfg(feature = "mock-versions")] @@ -69,7 +71,7 @@ pub static PLATFORM_TEST_VERSIONS: OnceLock> = OnceLock::ne #[cfg(feature = "mock-versions")] const DEFAULT_PLATFORM_TEST_VERSIONS: &[PlatformVersion] = &[TEST_PLATFORM_V2, TEST_PLATFORM_V3]; -pub const LATEST_PLATFORM_VERSION: &PlatformVersion = &PLATFORM_V14; +pub const LATEST_PLATFORM_VERSION: &PlatformVersion = &PLATFORM_V15; pub const DESIRED_PLATFORM_VERSION: &PlatformVersion = LATEST_PLATFORM_VERSION; diff --git a/packages/rs-platform-version/src/version/v1.rs b/packages/rs-platform-version/src/version/v1.rs index b3c54787c77..9a81628e63c 100644 --- a/packages/rs-platform-version/src/version/v1.rs +++ b/packages/rs-platform-version/src/version/v1.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v1::DRIVE_ABCI_METHOD_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v1::DRIVE_ABCI_VALIDATION_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -40,6 +41,7 @@ pub const PLATFORM_V1: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v10.rs b/packages/rs-platform-version/src/version/v10.rs index f04b14d341a..66eabb8d84e 100644 --- a/packages/rs-platform-version/src/version/v10.rs +++ b/packages/rs-platform-version/src/version/v10.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v6::DRIVE_ABCI_METHOD_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v6::DRIVE_ABCI_VALIDATION_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V10: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v11.rs b/packages/rs-platform-version/src/version/v11.rs index eb039ad49cf..414326d77a3 100644 --- a/packages/rs-platform-version/src/version/v11.rs +++ b/packages/rs-platform-version/src/version/v11.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v7::DRIVE_ABCI_METHOD_VERSIONS_V7; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v7::DRIVE_ABCI_VALIDATION_VERSIONS_V7; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V11: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v12.rs b/packages/rs-platform-version/src/version/v12.rs index 2d334b1fd7d..cac54b6cb49 100644 --- a/packages/rs-platform-version/src/version/v12.rs +++ b/packages/rs-platform-version/src/version/v12.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v8::DRIVE_ABCI_METHOD_VERSIONS_V8; use crate::version::drive_abci_versions::drive_abci_query_versions::v1::DRIVE_ABCI_QUERY_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v8::DRIVE_ABCI_VALIDATION_VERSIONS_V8; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -44,6 +45,7 @@ pub const PLATFORM_V12: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V1, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v13.rs b/packages/rs-platform-version/src/version/v13.rs index b6249ba91fc..776ad91a2e1 100644 --- a/packages/rs-platform-version/src/version/v13.rs +++ b/packages/rs-platform-version/src/version/v13.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v9::DRIVE_ABCI_METHOD_VERSIONS_V9; use crate::version::drive_abci_versions::drive_abci_query_versions::v1::DRIVE_ABCI_QUERY_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v9::DRIVE_ABCI_VALIDATION_VERSIONS_V9; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -70,6 +71,7 @@ pub const PLATFORM_V13: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V1, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v14.rs b/packages/rs-platform-version/src/version/v14.rs index 7589c485738..ddaf8397da3 100644 --- a/packages/rs-platform-version/src/version/v14.rs +++ b/packages/rs-platform-version/src/version/v14.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v10::DRIVE_ABCI_METHOD_VERSIONS_V10; use crate::version::drive_abci_versions::drive_abci_query_versions::v3::DRIVE_ABCI_QUERY_VERSIONS_V3; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v10::DRIVE_ABCI_VALIDATION_VERSIONS_V10; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v3::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3; @@ -204,6 +205,7 @@ pub const PLATFORM_V14: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3, // changed: prune bound for the total credits history query: DRIVE_ABCI_QUERY_VERSIONS_V3, // changed: ranked + boolean-HAVING routing gate; the v1 handler also resolves IN_TIME_RANGE from committed block time checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v15.rs b/packages/rs-platform-version/src/version/v15.rs new file mode 100644 index 00000000000..3080cb5ba83 --- /dev/null +++ b/packages/rs-platform-version/src/version/v15.rs @@ -0,0 +1,130 @@ +use crate::version::consensus_versions::ConsensusVersions; +use crate::version::dpp_versions::dpp_asset_lock_versions::v1::DPP_ASSET_LOCK_VERSIONS_V1; +use crate::version::dpp_versions::dpp_contract_versions::v6::CONTRACT_VERSIONS_V6; +use crate::version::dpp_versions::dpp_costs_versions::v1::DPP_COSTS_VERSIONS_V1; +use crate::version::dpp_versions::dpp_document_versions::v4::DOCUMENT_VERSIONS_V4; +use crate::version::dpp_versions::dpp_factory_versions::v1::DPP_FACTORY_VERSIONS_V1; +use crate::version::dpp_versions::dpp_identity_versions::v1::IDENTITY_VERSIONS_V1; +use crate::version::dpp_versions::dpp_method_versions::v3::DPP_METHOD_VERSIONS_V3; +use crate::version::dpp_versions::dpp_state_transition_conversion_versions::v2::STATE_TRANSITION_CONVERSION_VERSIONS_V2; +use crate::version::dpp_versions::dpp_state_transition_method_versions::v1::STATE_TRANSITION_METHOD_VERSIONS_V1; +use crate::version::dpp_versions::dpp_state_transition_serialization_versions::v3::STATE_TRANSITION_SERIALIZATION_VERSIONS_V3; +use crate::version::dpp_versions::dpp_state_transition_versions::v3::STATE_TRANSITION_VERSIONS_V3; +use crate::version::dpp_versions::dpp_token_versions::v2::TOKEN_VERSIONS_V2; +use crate::version::dpp_versions::dpp_validation_versions::v5::DPP_VALIDATION_VERSIONS_V5; +use crate::version::dpp_versions::dpp_voting_versions::v2::VOTING_VERSION_V2; +use crate::version::dpp_versions::DPPVersion; +use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; +use crate::version::drive_abci_versions::drive_abci_method_versions::v11::DRIVE_ABCI_METHOD_VERSIONS_V11; +use crate::version::drive_abci_versions::drive_abci_query_versions::v3::DRIVE_ABCI_QUERY_VERSIONS_V3; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_validation_versions::v10::DRIVE_ABCI_VALIDATION_VERSIONS_V10; +use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v3::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3; +use crate::version::drive_abci_versions::DriveAbciVersion; +use crate::version::drive_versions::v9::DRIVE_VERSION_V9; +use crate::version::fee::v2::FEE_VERSION2; +use crate::version::protocol_version::PlatformVersion; +use crate::version::system_data_contract_versions::v3::SYSTEM_DATA_CONTRACT_VERSIONS_V3; +use crate::version::system_limits::v4::SYSTEM_LIMITS_V4; +use crate::version::ProtocolVersion; + +pub const PROTOCOL_VERSION_15: ProtocolVersion = 15; + +/// v15 enables ABCI state sync: a fresh node can bootstrap from a peer's grovedb +/// snapshot instead of replaying the chain. +/// +/// The consensus changes gate on `DRIVE_ABCI_METHOD_VERSIONS_V11`: +/// +/// * `run_block_proposal` 0 -> 1: every block writes a reduced platform state +/// (`Misc/reduced_saved_state`) into the replicated state just before the root hash is +/// computed, and `validator_set_update` moves above the root-hash computation so the +/// stored reduced state reflects the post-rotation validator set. The full platform +/// state only lives in non-replicated aux storage, so without this a state-synced node +/// would have no way to rebuild its in-memory state. +/// * `consensus_params_update` 1 -> 2: the first block of v15 also emits evidence +/// params sized for state-synced nodes that do not hold full history (issue #2512). +/// * The activation block already runs `run_block_proposal` v1, so the reduced state +/// exists from that block on and every snapshot taken at or after activation is +/// restorable. Snapshots from before activation lack the key and are not served. +/// +/// Everything else matches v14. The grovedb state sync protocol version used for +/// snapshots is `DRIVE_ABCI_STATE_SYNC_VERSIONS_V1.protocol_version` (1), shared by all +/// platform versions; grovedb updates its replication protocol in place, so exactly one +/// version exists. +pub const PLATFORM_V15: PlatformVersion = PlatformVersion { + protocol_version: PROTOCOL_VERSION_15, + drive: DRIVE_VERSION_V9, + drive_abci: DriveAbciVersion { + structs: DRIVE_ABCI_STRUCTURE_VERSIONS_V1, + methods: DRIVE_ABCI_METHOD_VERSIONS_V11, // changed: run_block_proposal v1 (reduced state write + validator rotation above root hash) and consensus_params_update v2 (evidence params on the v15 activation block) + validation_and_processing: DRIVE_ABCI_VALIDATION_VERSIONS_V10, + withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3, + query: DRIVE_ABCI_QUERY_VERSIONS_V3, + checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, + }, + dpp: DPPVersion { + costs: DPP_COSTS_VERSIONS_V1, + validation: DPP_VALIDATION_VERSIONS_V5, + state_transition_serialization_versions: STATE_TRANSITION_SERIALIZATION_VERSIONS_V3, + state_transition_conversion_versions: STATE_TRANSITION_CONVERSION_VERSIONS_V2, + state_transition_method_versions: STATE_TRANSITION_METHOD_VERSIONS_V1, + state_transitions: STATE_TRANSITION_VERSIONS_V3, + contract_versions: CONTRACT_VERSIONS_V6, + document_versions: DOCUMENT_VERSIONS_V4, + identity_versions: IDENTITY_VERSIONS_V1, + voting_versions: VOTING_VERSION_V2, + token_versions: TOKEN_VERSIONS_V2, + asset_lock_versions: DPP_ASSET_LOCK_VERSIONS_V1, + methods: DPP_METHOD_VERSIONS_V3, + factory_versions: DPP_FACTORY_VERSIONS_V1, + }, + system_data_contracts: SYSTEM_DATA_CONTRACT_VERSIONS_V3, + fee_version: FEE_VERSION2, + system_limits: SYSTEM_LIMITS_V4, + consensus: ConsensusVersions { + tenderdash_consensus_version: 1, + }, +}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::version::v14::PLATFORM_V14; + + /// The state sync consensus changes live in v15's own method table, so a v14 node + /// keeps running run_block_proposal v0 (no reduced-state write, rotation after the + /// root hash) and consensus_params_update v1. Making v14 non-zero here would be + /// consensus-breaking for already-deployed nodes. + #[test] + fn state_sync_consensus_changes_gate_at_v15() { + assert_eq!(PLATFORM_V14.drive_abci.methods.engine.run_block_proposal, 0); + assert_eq!( + PLATFORM_V14 + .drive_abci + .methods + .engine + .consensus_params_update, + 1 + ); + assert_eq!(PLATFORM_V15.drive_abci.methods.engine.run_block_proposal, 1); + assert_eq!( + PLATFORM_V15 + .drive_abci + .methods + .engine + .consensus_params_update, + 2 + ); + } + + /// All platform versions share grovedb state sync protocol version 1 — the only + /// version that exists, since grovedb updates its replication protocol in place. + /// The supported set lives next to the snapshot types in drive-abci. + #[test] + fn state_sync_wire_protocol_version_is_one() { + assert_eq!(PLATFORM_V15.drive_abci.state_sync.protocol_version, 1); + assert_eq!(PLATFORM_V14.drive_abci.state_sync.protocol_version, 1); + } +} diff --git a/packages/rs-platform-version/src/version/v2.rs b/packages/rs-platform-version/src/version/v2.rs index 93cd7b07232..0bcfcb9fbeb 100644 --- a/packages/rs-platform-version/src/version/v2.rs +++ b/packages/rs-platform-version/src/version/v2.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v1::DRIVE_ABCI_METHOD_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v2::DRIVE_ABCI_VALIDATION_VERSIONS_V2; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -40,6 +41,7 @@ pub const PLATFORM_V2: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v3.rs b/packages/rs-platform-version/src/version/v3.rs index c125b94ff9b..c8bfa8b212d 100644 --- a/packages/rs-platform-version/src/version/v3.rs +++ b/packages/rs-platform-version/src/version/v3.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v2::DRIVE_ABCI_METHOD_VERSIONS_V2; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v2::DRIVE_ABCI_VALIDATION_VERSIONS_V2; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -46,6 +47,7 @@ pub const PLATFORM_V3: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v4.rs b/packages/rs-platform-version/src/version/v4.rs index dba41251e96..c7268418092 100644 --- a/packages/rs-platform-version/src/version/v4.rs +++ b/packages/rs-platform-version/src/version/v4.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v3::DRIVE_ABCI_METHOD_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v3::DRIVE_ABCI_VALIDATION_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V4: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v5.rs b/packages/rs-platform-version/src/version/v5.rs index 3c288cfe63d..e0e9150dfd6 100644 --- a/packages/rs-platform-version/src/version/v5.rs +++ b/packages/rs-platform-version/src/version/v5.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v4::DRIVE_ABCI_METHOD_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v3::DRIVE_ABCI_VALIDATION_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V5: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v6.rs b/packages/rs-platform-version/src/version/v6.rs index 7d948da6f00..43def28162d 100644 --- a/packages/rs-platform-version/src/version/v6.rs +++ b/packages/rs-platform-version/src/version/v6.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v4::DRIVE_ABCI_METHOD_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v4::DRIVE_ABCI_VALIDATION_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V6: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v7.rs b/packages/rs-platform-version/src/version/v7.rs index 09755d462e1..eabdc58c585 100644 --- a/packages/rs-platform-version/src/version/v7.rs +++ b/packages/rs-platform-version/src/version/v7.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v4::DRIVE_ABCI_METHOD_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v5::DRIVE_ABCI_VALIDATION_VERSIONS_V5; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V7: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v8.rs b/packages/rs-platform-version/src/version/v8.rs index 2096142ac18..f4c2f9dd1f5 100644 --- a/packages/rs-platform-version/src/version/v8.rs +++ b/packages/rs-platform-version/src/version/v8.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v5::DRIVE_ABCI_METHOD_VERSIONS_V5; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v5::DRIVE_ABCI_VALIDATION_VERSIONS_V5; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -45,6 +46,7 @@ pub const PLATFORM_V8: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v9.rs b/packages/rs-platform-version/src/version/v9.rs index a27803f6da8..8ee4fc891bf 100644 --- a/packages/rs-platform-version/src/version/v9.rs +++ b/packages/rs-platform-version/src/version/v9.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v6::DRIVE_ABCI_METHOD_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v6::DRIVE_ABCI_VALIDATION_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V9: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1,