From 6a828836e6e3a257fcfd2c5f69ba8e28ac6fef1f Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Mon, 8 Jul 2024 03:33:36 -0700 Subject: [PATCH 1/6] Print genesis block from config when there is mismatch between genesis in storage and config. --- chain/chain/src/chain.rs | 49 +++++---------- chain/chain/src/runtime/tests.rs | 11 +++- integration-tests/src/tests/network/runner.rs | 12 +++- nearcore/src/lib.rs | 10 +++- tools/state-viewer/src/cli.rs | 20 +++++++ tools/state-viewer/src/commands.rs | 59 ++++++++++++++++++- 6 files changed, 119 insertions(+), 42 deletions(-) diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index 7d9351a1785..bff871debdd 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -314,13 +314,13 @@ pub enum VerifyBlockHashAndSignatureResult { } impl Chain { + /// Builds genesis block and chunks from the current configuration obtained through the arguments. pub fn make_genesis_block( epoch_manager: &dyn EpochManagerAdapter, runtime_adapter: &dyn RuntimeAdapter, chain_genesis: &ChainGenesis, - ) -> Result { - let state_roots = get_genesis_state_roots(runtime_adapter.store())? - .expect("genesis should be initialized."); + state_roots: Vec, + ) -> Result<(Block, Vec), Error> { let congestion_infos = get_genesis_congestion_infos(epoch_manager, runtime_adapter, &state_roots)?; let genesis_chunks = genesis_chunks( @@ -331,9 +331,9 @@ impl Chain { chain_genesis.height, chain_genesis.protocol_version, ); - Ok(Block::genesis( + let genesis_block = Block::genesis( chain_genesis.protocol_version, - genesis_chunks.into_iter().map(|chunk| chunk.take_header()).collect(), + genesis_chunks.iter().map(|chunk| chunk.cloned_header()).collect(), chain_genesis.time, chain_genesis.height, chain_genesis.min_gas_price, @@ -344,7 +344,8 @@ impl Chain { EpochId::default(), &CryptoHash::default(), )?, - )) + ); + Ok((genesis_block, genesis_chunks)) } pub fn new_for_view_client( @@ -358,10 +359,13 @@ impl Chain { ) -> Result { let store = runtime_adapter.store(); let chain_store = ChainStore::new(store.clone(), chain_genesis.height, save_trie_changes); - let genesis = Self::make_genesis_block( + let state_roots = get_genesis_state_roots(runtime_adapter.store())? + .expect("genesis should be initialized."); + let (genesis, _genesis_chunks) = Self::make_genesis_block( epoch_manager.as_ref(), runtime_adapter.as_ref(), chain_genesis, + state_roots, )?; let (sc, rc) = unbounded(); Ok(Chain { @@ -407,44 +411,21 @@ impl Chain { apply_chunks_spawner: Arc, validator: MutableValidatorSigner, ) -> Result { - // Get runtime initial state and create genesis block out of it. let state_roots = get_genesis_state_roots(runtime_adapter.store())? .expect("genesis should be initialized."); - let congestion_infos = get_genesis_congestion_infos( + let (genesis, genesis_chunks) = Self::make_genesis_block( epoch_manager.as_ref(), runtime_adapter.as_ref(), - &state_roots, - )?; - let genesis_chunks = genesis_chunks( + chain_genesis, state_roots.clone(), - congestion_infos, - &epoch_manager.shard_ids(&EpochId::default())?, - chain_genesis.gas_limit, - chain_genesis.height, - chain_genesis.protocol_version, - ); - let genesis = Block::genesis( - chain_genesis.protocol_version, - genesis_chunks.iter().map(|chunk| chunk.cloned_header()).collect(), - chain_genesis.time, - chain_genesis.height, - chain_genesis.min_gas_price, - chain_genesis.total_supply, - Chain::compute_bp_hash( - epoch_manager.as_ref(), - EpochId::default(), - EpochId::default(), - &CryptoHash::default(), - )?, - ); + )?; + // Check if we have a head in the store, otherwise pick genesis block. let mut chain_store = ChainStore::new( runtime_adapter.store().clone(), chain_genesis.height, chain_config.save_trie_changes, ); - - // Check if we have a head in the store, otherwise pick genesis block. let mut store_update = chain_store.store_update(); let (block_head, header_head) = match store_update.head() { Ok(block_head) => { diff --git a/chain/chain/src/runtime/tests.rs b/chain/chain/src/runtime/tests.rs index 776ef292840..c43ec60888e 100644 --- a/chain/chain/src/runtime/tests.rs +++ b/chain/chain/src/runtime/tests.rs @@ -1534,8 +1534,15 @@ fn test_genesis_hash() { StateSnapshotType::EveryEpoch, ); - let block = Chain::make_genesis_block(epoch_manager.as_ref(), runtime.as_ref(), &chain_genesis) - .unwrap(); + let state_roots = + get_genesis_state_roots(runtime_adapter.store())?.expect("genesis should be initialized."); + let (block, _chunks) = Chain::make_genesis_block( + epoch_manager.as_ref(), + runtime.as_ref(), + &chain_genesis, + state_roots, + ) + .unwrap(); assert_eq!(block.header().hash().to_string(), "EPnLgE7iEq9s7yTkos96M3cWymH5avBAPm3qx3NXqR8H"); let epoch_manager = EpochManager::new_from_genesis_config(store, &genesis.config).unwrap(); diff --git a/integration-tests/src/tests/network/runner.rs b/integration-tests/src/tests/network/runner.rs index 9c7ab737c74..efef4a926be 100644 --- a/integration-tests/src/tests/network/runner.rs +++ b/integration-tests/src/tests/network/runner.rs @@ -77,9 +77,15 @@ fn setup_network_node( ClientConfig::test(false, 100, 200, num_validators, false, true, true, true); client_config.archive = config.archive; client_config.ttl_account_id_router = config.ttl_account_id_router.try_into().unwrap(); - let genesis_block = - Chain::make_genesis_block(epoch_manager.as_ref(), runtime.as_ref(), &chain_genesis) - .unwrap(); + let state_roots = + get_genesis_state_roots(runtime_adapter.store())?.expect("genesis should be initialized."); + let (genesis_block, _genesis_chunks) = Chain::make_genesis_block( + epoch_manager.as_ref(), + runtime.as_ref(), + &chain_genesis, + state_roots, + ) + .unwrap(); let genesis_id = GenesisId { chain_id: client_config.chain_id.clone(), hash: *genesis_block.header().hash(), diff --git a/nearcore/src/lib.rs b/nearcore/src/lib.rs index 00879b25c26..3719d05dc0b 100644 --- a/nearcore/src/lib.rs +++ b/nearcore/src/lib.rs @@ -306,8 +306,14 @@ pub fn start_with_config_and_synchronization( let telemetry = ActixWrapper::new(TelemetryActor::new(config.telemetry_config.clone())).start(); let chain_genesis = ChainGenesis::new(&config.genesis.config); - let genesis_block = - Chain::make_genesis_block(epoch_manager.as_ref(), runtime.as_ref(), &chain_genesis)?; + let state_roots = near_store::get_genesis_state_roots(runtime.store())? + .expect("genesis should be initialized."); + let (genesis_block, _genesis_chunks) = Chain::make_genesis_block( + epoch_manager.as_ref(), + runtime.as_ref(), + &chain_genesis, + state_roots, + )?; let genesis_id = GenesisId { chain_id: config.client_config.chain_id.clone(), hash: *genesis_block.header().hash(), diff --git a/tools/state-viewer/src/cli.rs b/tools/state-viewer/src/cli.rs index 9a6e71924b7..303d335e8ee 100644 --- a/tools/state-viewer/src/cli.rs +++ b/tools/state-viewer/src/cli.rs @@ -97,6 +97,9 @@ pub enum StateViewerSubCommand { /// View head of the storage. #[clap(alias = "view_chain")] ViewChain(ViewChainCmd), + /// View genesis block and chunks built from the config and in the store. + #[clap(alias = "view_genesis")] + ViewGenesis(ViewGenesisCmd), /// View trie structure. #[clap(alias = "view_trie")] ViewTrie(ViewTrieCmd), @@ -172,6 +175,7 @@ impl StateViewerSubCommand { StateViewerSubCommand::StateParts(cmd) => cmd.run(home_dir, near_config, store), StateViewerSubCommand::StateStats(cmd) => cmd.run(home_dir, near_config, store), StateViewerSubCommand::ViewChain(cmd) => cmd.run(near_config, store), + StateViewerSubCommand::ViewGenesis(cmd) => cmd.run(home_dir, near_config, store), StateViewerSubCommand::ViewTrie(cmd) => cmd.run(store), StateViewerSubCommand::TrieIterationBenchmark(cmd) => cmd.run(near_config, store), StateViewerSubCommand::StateWitness(cmd) => cmd.run(home_dir, near_config, store), @@ -721,6 +725,22 @@ impl ViewChainCmd { } } +#[derive(clap::Parser)] +pub struct ViewGenesisCmd { + /// If true, displays the genesis block built from the config. + #[clap(long)] + view_config: bool, + /// If true, displays the genesis block saved in the store. + #[clap(long)] + view_store: bool, +} + +impl ViewGenesisCmd { + pub fn run(self, home_dir: &Path, near_config: NearConfig, store: Store) { + view_genesis(home_dir, near_config, store, self.view_config, self.view_store); + } +} + #[derive(Clone)] pub enum ViewTrieFormat { Full, diff --git a/tools/state-viewer/src/commands.rs b/tools/state-viewer/src/commands.rs index f81b40c9f86..7d890d7aff8 100644 --- a/tools/state-viewer/src/commands.rs +++ b/tools/state-viewer/src/commands.rs @@ -19,7 +19,7 @@ use near_chain::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, RuntimeAdapter, RuntimeStorageConfig, }; -use near_chain::{ChainStore, ChainStoreAccess, ChainStoreUpdate, Error}; +use near_chain::{Chain, ChainGenesis, ChainStore, ChainStoreAccess, ChainStoreUpdate, Error}; use near_chain_configs::GenesisChangeConfig; use near_epoch_manager::types::BlockHeaderInfo; use near_epoch_manager::EpochManagerHandle; @@ -846,6 +846,63 @@ pub(crate) fn view_chain( } } +pub(crate) fn view_genesis( + home_dir: &Path, + near_config: NearConfig, + store: Store, + view_config: bool, + view_store: bool, +) { + let chain_genesis = ChainGenesis::new(&near_config.genesis.config); + let epoch_manager = EpochManager::new_arc_handle(store.clone(), &near_config.genesis.config); + let runtime_adapter = NightshadeRuntime::from_config( + home_dir, + store.clone(), + &near_config, + epoch_manager.clone(), + ) + .unwrap(); + let chain_store = ChainStore::new( + store, + near_config.genesis.config.genesis_height, + near_config.client_config.save_trie_changes, + ); + + if view_config { + let state_roots = + near_store::get_genesis_state_roots(chain_store.store()).unwrap().unwrap(); + let (genesis_block, genesis_chunks) = Chain::make_genesis_block( + epoch_manager.as_ref(), + runtime_adapter.as_ref(), + &chain_genesis, + state_roots, + ) + .unwrap(); + + println!("Genesis block from config: {:#?}", genesis_block); + for chunk in genesis_chunks { + println!("Genesis chunk from config at shard {}: {:#?}", chunk.shard_id(), chunk); + } + + // Check that genesis in the store is the same as genesis given in the config. + let genesis_hash_in_storage = + chain_store.get_block_hash_by_height(chain_genesis.height).unwrap(); + let genesis_hash_in_config = genesis_block.hash(); + if &genesis_hash_in_storage == genesis_hash_in_config { + println!("Genesis in storage and config match."); + } else { + println!( + "Genesis mismatch between storage and config: {:?} vs {:?}", + genesis_hash_in_storage, genesis_hash_in_config + ); + } + } + + if view_store { + unimplemented!("Viewing genesis from config is not yet implemented") + } +} + pub(crate) fn check_block_chunk_existence(near_config: NearConfig, store: Store) { let genesis_height = near_config.genesis.config.genesis_height; let chain_store = From e8f2230bb90db3eb1c1d5812d8ec8685f2eac2dd Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Mon, 8 Jul 2024 05:06:49 -0700 Subject: [PATCH 2/6] Fix clippy findings. --- chain/chain/src/runtime/tests.rs | 2 +- integration-tests/src/tests/network/runner.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/chain/chain/src/runtime/tests.rs b/chain/chain/src/runtime/tests.rs index c43ec60888e..e0c63594d3f 100644 --- a/chain/chain/src/runtime/tests.rs +++ b/chain/chain/src/runtime/tests.rs @@ -1535,7 +1535,7 @@ fn test_genesis_hash() { ); let state_roots = - get_genesis_state_roots(runtime_adapter.store())?.expect("genesis should be initialized."); + get_genesis_state_roots(runtime.store()).unwrap().expect("genesis should be initialized."); let (block, _chunks) = Chain::make_genesis_block( epoch_manager.as_ref(), runtime.as_ref(), diff --git a/integration-tests/src/tests/network/runner.rs b/integration-tests/src/tests/network/runner.rs index efef4a926be..9a3979b2e38 100644 --- a/integration-tests/src/tests/network/runner.rs +++ b/integration-tests/src/tests/network/runner.rs @@ -77,8 +77,9 @@ fn setup_network_node( ClientConfig::test(false, 100, 200, num_validators, false, true, true, true); client_config.archive = config.archive; client_config.ttl_account_id_router = config.ttl_account_id_router.try_into().unwrap(); - let state_roots = - get_genesis_state_roots(runtime_adapter.store())?.expect("genesis should be initialized."); + let state_roots = near_store::get_genesis_state_roots(runtime.store()) + .unwrap() + .expect("genesis should be initialized."); let (genesis_block, _genesis_chunks) = Chain::make_genesis_block( epoch_manager.as_ref(), runtime.as_ref(), From c9464d91559b78c031be3d37fc5fd9871627a38f Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Wed, 17 Jul 2024 03:05:55 -0700 Subject: [PATCH 3/6] Address comments. --- tools/state-viewer/src/cli.rs | 15 +++++++++++--- tools/state-viewer/src/commands.rs | 33 +++++++++++++++++------------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/tools/state-viewer/src/cli.rs b/tools/state-viewer/src/cli.rs index cde4ba733b4..0f51bcf0922 100644 --- a/tools/state-viewer/src/cli.rs +++ b/tools/state-viewer/src/cli.rs @@ -743,17 +743,26 @@ impl ViewChainCmd { #[derive(clap::Parser)] pub struct ViewGenesisCmd { - /// If true, displays the genesis block built from the config. + /// If true, displays the genesis block built from nearcore code that combines the + /// contents of the genesis config (JSON) file with some hard-coded logic to set some + /// fields of the genesis block. At any given time, the block built this way should match + /// the genesis block recorded in the store (to be displayed with the --view-store option). #[clap(long)] view_config: bool, - /// If true, displays the genesis block saved in the store. + /// If true, displays the genesis block saved in the store, when the genesis block is built + /// for the first time. At any given time, this saved block should match the genesis block + /// built by the code (to be displayed with the --view-config option). #[clap(long)] view_store: bool, + /// If true, compares the contents of the genesis block saved in the store with + /// the genesis block built from the genesis config (JSON) file. + #[clap(long, default_value = "false")] + compare: bool, } impl ViewGenesisCmd { pub fn run(self, home_dir: &Path, near_config: NearConfig, store: Store) { - view_genesis(home_dir, near_config, store, self.view_config, self.view_store); + view_genesis(home_dir, near_config, store, self.view_config, self.view_store, self.compare); } } diff --git a/tools/state-viewer/src/commands.rs b/tools/state-viewer/src/commands.rs index a2a2a85f71f..0546ae59eac 100644 --- a/tools/state-viewer/src/commands.rs +++ b/tools/state-viewer/src/commands.rs @@ -871,6 +871,7 @@ pub(crate) fn view_genesis( store: Store, view_config: bool, view_store: bool, + compare: bool, ) { let chain_genesis = ChainGenesis::new(&near_config.genesis.config); let epoch_manager = EpochManager::new_arc_handle(store.clone(), &near_config.genesis.config); @@ -887,7 +888,7 @@ pub(crate) fn view_genesis( near_config.client_config.save_trie_changes, ); - if view_config { + if view_config || compare { let state_roots = near_store::get_genesis_state_roots(chain_store.store()).unwrap().unwrap(); let (genesis_block, genesis_chunks) = Chain::make_genesis_block( @@ -898,22 +899,26 @@ pub(crate) fn view_genesis( ) .unwrap(); - println!("Genesis block from config: {:#?}", genesis_block); - for chunk in genesis_chunks { - println!("Genesis chunk from config at shard {}: {:#?}", chunk.shard_id(), chunk); + if view_config { + println!("Genesis block from config: {:#?}", genesis_block); + for chunk in genesis_chunks { + println!("Genesis chunk from config at shard {}: {:#?}", chunk.shard_id(), chunk); + } } // Check that genesis in the store is the same as genesis given in the config. - let genesis_hash_in_storage = - chain_store.get_block_hash_by_height(chain_genesis.height).unwrap(); - let genesis_hash_in_config = genesis_block.hash(); - if &genesis_hash_in_storage == genesis_hash_in_config { - println!("Genesis in storage and config match."); - } else { - println!( - "Genesis mismatch between storage and config: {:?} vs {:?}", - genesis_hash_in_storage, genesis_hash_in_config - ); + if compare { + let genesis_hash_in_storage = + chain_store.get_block_hash_by_height(chain_genesis.height).unwrap(); + let genesis_hash_in_config = genesis_block.hash(); + if &genesis_hash_in_storage == genesis_hash_in_config { + println!("Genesis in storage and config match."); + } else { + println!( + "Genesis mismatch between storage and config: {:?} vs {:?}", + genesis_hash_in_storage, genesis_hash_in_config + ); + } } } From aac6fdce7e37dca5d28b15d6205a25f738f31b55 Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Thu, 18 Jul 2024 04:17:06 -0700 Subject: [PATCH 4/6] Implement view-store --- tools/state-viewer/src/commands.rs | 46 +++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/tools/state-viewer/src/commands.rs b/tools/state-viewer/src/commands.rs index 0546ae59eac..37662502a96 100644 --- a/tools/state-viewer/src/commands.rs +++ b/tools/state-viewer/src/commands.rs @@ -30,7 +30,7 @@ use near_primitives::epoch_manager::epoch_info::EpochInfo; use near_primitives::hash::CryptoHash; use near_primitives::shard_layout::ShardLayout; use near_primitives::shard_layout::ShardUId; -use near_primitives::sharding::ChunkHash; +use near_primitives::sharding::{ChunkHash, ShardChunk}; use near_primitives::state::FlatStateValue; use near_primitives::state_record::state_record_to_account_id; use near_primitives::state_record::StateRecord; @@ -882,13 +882,12 @@ pub(crate) fn view_genesis( epoch_manager.clone(), ) .unwrap(); - let chain_store = ChainStore::new( - store, - near_config.genesis.config.genesis_height, - near_config.client_config.save_trie_changes, - ); + let genesis_height = near_config.genesis.config.genesis_height; + let chain_store = + ChainStore::new(store.clone(), genesis_height, near_config.client_config.save_trie_changes); if view_config || compare { + tracing::info!(target: "state_viewer", "Computing genesis from config..."); let state_roots = near_store::get_genesis_state_roots(chain_store.store()).unwrap().unwrap(); let (genesis_block, genesis_chunks) = Chain::make_genesis_block( @@ -923,8 +922,41 @@ pub(crate) fn view_genesis( } if view_store { - unimplemented!("Viewing genesis from config is not yet implemented") + tracing::info!(target: "state_viewer", genesis_height, "Reading genesis from store..."); + match read_genesis_from_store(&chain_store, genesis_height) { + Ok((genesis_block, genesis_chunks)) => { + println!("Genesis block from store: {:#?}", genesis_block); + for chunk in genesis_chunks { + println!( + "Genesis chunk from store at shard {}: {:#?}", + chunk.shard_id(), + chunk + ); + } + } + Err(error) => { + println!("Failed to read genesis block from store. Error: {}", error); + if !near_config.config.archive { + println!("Hint: This is not an archival node. Try running this command from an archival node since genesis block may be garbage collected."); + } + } + } + } +} + +fn read_genesis_from_store( + chain_store: &ChainStore, + genesis_height: u64, +) -> Result<(Block, Vec>), Error> { + let genesis_hash = chain_store.get_block_hash_by_height(genesis_height)?; + let genesis_block = chain_store.get_block(&genesis_hash)?; + let mut genesis_chunks = vec![]; + for chunk_header in genesis_block.chunks().iter() { + if chunk_header.height_included() == genesis_height { + genesis_chunks.push(chain_store.get_chunk(&chunk_header.chunk_hash())?); + } } + Ok((genesis_block, genesis_chunks)) } pub(crate) fn check_block_chunk_existence(near_config: NearConfig, store: Store) { From 8b5da2ddce77756c7eafd2edc7b3407869fe8854 Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Thu, 18 Jul 2024 04:27:28 -0700 Subject: [PATCH 5/6] Shorten command flags --- tools/state-viewer/src/cli.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/state-viewer/src/cli.rs b/tools/state-viewer/src/cli.rs index 0f51bcf0922..7e72af69154 100644 --- a/tools/state-viewer/src/cli.rs +++ b/tools/state-viewer/src/cli.rs @@ -746,14 +746,14 @@ pub struct ViewGenesisCmd { /// If true, displays the genesis block built from nearcore code that combines the /// contents of the genesis config (JSON) file with some hard-coded logic to set some /// fields of the genesis block. At any given time, the block built this way should match - /// the genesis block recorded in the store (to be displayed with the --view-store option). + /// the genesis block recorded in the store (to be displayed with the --store option). #[clap(long)] - view_config: bool, + config: bool, /// If true, displays the genesis block saved in the store, when the genesis block is built /// for the first time. At any given time, this saved block should match the genesis block - /// built by the code (to be displayed with the --view-config option). + /// built by the code (to be displayed with the --config option). #[clap(long)] - view_store: bool, + store: bool, /// If true, compares the contents of the genesis block saved in the store with /// the genesis block built from the genesis config (JSON) file. #[clap(long, default_value = "false")] @@ -762,7 +762,7 @@ pub struct ViewGenesisCmd { impl ViewGenesisCmd { pub fn run(self, home_dir: &Path, near_config: NearConfig, store: Store) { - view_genesis(home_dir, near_config, store, self.view_config, self.view_store, self.compare); + view_genesis(home_dir, near_config, store, self.config, self.store, self.compare); } } From a2686daae4d0447da9e397bc1a3e04d6ffbf1be1 Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Thu, 18 Jul 2024 05:50:19 -0700 Subject: [PATCH 6/6] fix clippy finding --- tools/state-viewer/src/commands.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/state-viewer/src/commands.rs b/tools/state-viewer/src/commands.rs index 37662502a96..51e400c5b9b 100644 --- a/tools/state-viewer/src/commands.rs +++ b/tools/state-viewer/src/commands.rs @@ -884,7 +884,7 @@ pub(crate) fn view_genesis( .unwrap(); let genesis_height = near_config.genesis.config.genesis_height; let chain_store = - ChainStore::new(store.clone(), genesis_height, near_config.client_config.save_trie_changes); + ChainStore::new(store, genesis_height, near_config.client_config.save_trie_changes); if view_config || compare { tracing::info!(target: "state_viewer", "Computing genesis from config...");