From 5e7019a1b25d02f711458dc69952296aeaa89eba Mon Sep 17 00:00:00 2001 From: Vasily Selivanov Date: Tue, 1 Aug 2023 13:24:54 +0300 Subject: [PATCH] Fixed uploading for big trees (#528) * fixed uploading for big trees * bump up version --- v4_x/git-remote-gosh/Cargo.toml | 2 +- .../git-remote-gosh/src/blockchain/service.rs | 3 +- .../src/blockchain/tree/save.rs | 53 ++++++++++-------- .../push/parallel_snapshot_upload_support.rs | 9 ++-- .../src/git_helper/push/push_tree.rs | 5 +- v5_x/v5.0.0/git-remote-gosh/Cargo.toml | 2 +- .../git-remote-gosh/src/blockchain/service.rs | 3 +- .../src/blockchain/tree/save.rs | 54 +++++++++++-------- .../push/parallel_snapshot_upload_support.rs | 8 ++- .../src/git_helper/push/push_tree.rs | 5 +- v5_x/v5.1.0/git-remote-gosh/Cargo.toml | 2 +- .../git-remote-gosh/src/blockchain/service.rs | 3 +- .../src/blockchain/tree/save.rs | 54 +++++++++++-------- .../push/parallel_snapshot_upload_support.rs | 8 ++- .../src/git_helper/push/push_tree.rs | 5 +- version.json | 2 +- 16 files changed, 124 insertions(+), 94 deletions(-) diff --git a/v4_x/git-remote-gosh/Cargo.toml b/v4_x/git-remote-gosh/Cargo.toml index b4df224a8..3aac2e548 100644 --- a/v4_x/git-remote-gosh/Cargo.toml +++ b/v4_x/git-remote-gosh/Cargo.toml @@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh'] name = 'git-remote-gosh' resolver = '2' rust-version = "1.68" -version = '4.1.26' +version = '4.1.29' [profile.profiling] debug = 1 diff --git a/v4_x/git-remote-gosh/src/blockchain/service.rs b/v4_x/git-remote-gosh/src/blockchain/service.rs index 4b310e54b..40769c6e0 100644 --- a/v4_x/git-remote-gosh/src/blockchain/service.rs +++ b/v4_x/git-remote-gosh/src/blockchain/service.rs @@ -213,8 +213,9 @@ pub mod tests { &self, wallet: &UserWallet, sha: &str, + tree_address: &str, repo_name: &str, - nodes: &HashMap, + nodes: &mut HashMap, ) -> anyhow::Result<()>; } diff --git a/v4_x/git-remote-gosh/src/blockchain/tree/save.rs b/v4_x/git-remote-gosh/src/blockchain/tree/save.rs index 5530395b2..4417e3095 100644 --- a/v4_x/git-remote-gosh/src/blockchain/tree/save.rs +++ b/v4_x/git-remote-gosh/src/blockchain/tree/save.rs @@ -1,8 +1,6 @@ -use crate::blockchain::contract::ContractInfo; -use crate::blockchain::user_wallet::UserWallet; use crate::blockchain::{ - call::BlockchainCall, BlockchainContractAddress, BlockchainService, Everscale, - GoshBlobBitFlags, Tree, + self, call::BlockchainCall, contract::ContractInfo, gosh_abi, user_wallet::UserWallet, + BlockchainContractAddress, BlockchainService, GoshContract, Everscale, GoshBlobBitFlags, Tree, }; use async_trait::async_trait; use git_object; @@ -59,12 +57,13 @@ pub trait DeployTree { &self, wallet: &UserWallet, sha: &str, + tree_address: &str, repo_name: &str, - nodes: &HashMap, + nodes: &mut HashMap, ) -> anyhow::Result<()>; } -static TREE_NODES_CHUNK_MAX_SIZE: usize = 200; +static TREE_NODES_CHUNK_MAX_SIZE: usize = 50; #[async_trait] impl DeployTree for Everscale { @@ -72,8 +71,9 @@ impl DeployTree for Everscale { &self, wallet: &UserWallet, sha: &str, + tree_address: &str, repo_name: &str, - nodes: &HashMap, // change to moved hashmap + nodes: &mut HashMap, // change to moved hashmap ) -> anyhow::Result<()> { let wallet_contract = wallet.take_one().await?; tracing::trace!("Acquired wallet: {}", wallet_contract.get_address()); @@ -86,21 +86,30 @@ impl DeployTree for Everscale { .await?; let mut nodes = nodes.to_owned(); let chunk: HashMap = HashMap::new(); - let params = DeployTreeArgs { - sha: sha.to_owned(), - repo_name: repo_name.to_owned(), - nodes: chunk, - number: nodes_cnt as u128, - }; - tracing::trace!("DeployTreeArgs: {params:?}"); - self.send_message( - wallet_contract.deref(), - "deployTree", - Some(serde_json::to_value(params.clone())?), - None, - ) - .await - .map(|_| ())?; + let tree_contract = + GoshContract::new(BlockchainContractAddress::new(&tree_address), gosh_abi::TREE); + + if tree_contract.is_active(self.client()).await? { + let onchain_tree_object = + blockchain::Tree::load(self.client(), &tree_address).await?; + nodes.retain(|k, _| !onchain_tree_object.objects.contains_key(k)); + } else { + let params = DeployTreeArgs { + sha: sha.to_owned(), + repo_name: repo_name.to_owned(), + nodes: chunk, + number: nodes_cnt as u128, + }; + tracing::trace!("DeployTreeArgs: {params:?}"); + self.send_message( + wallet_contract.deref(), + "deployTree", + Some(serde_json::to_value(params.clone())?), + None, + ) + .await + .map(|_| ())?; + } while nodes.len() > 0 { let mut counter = 0; let chunk: HashMap; diff --git a/v4_x/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs b/v4_x/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs index 8522fe3dc..2334d8953 100644 --- a/v4_x/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs +++ b/v4_x/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs @@ -1,8 +1,9 @@ -use crate::blockchain::contract::wait_contracts_deployed::wait_contracts_deployed; -use crate::blockchain::tree::load::check_if_tree_is_ready; use crate::{ blockchain::{ - tree::TreeNode, user_wallet::WalletError, AddrVersion, BlockchainContractAddress, + contract::{wait_contracts_deployed::wait_contracts_deployed, GoshContract}, + gosh_abi, + tree::{load::check_if_tree_is_ready, TreeNode}, + user_wallet::WalletError, AddrVersion, BlockchainContractAddress, BlockchainService, }, git_helper::{ @@ -330,8 +331,6 @@ impl ParallelTreeUploadSupport { let remote_network = context.remote.network.clone(); let repo = context.remote.repo.clone(); - tracing::trace!("Start push of tree: address: {tree_address:?}"); - self.expecting_deployed_contacts_addresses .push(tree_address.clone()); diff --git a/v4_x/git-remote-gosh/src/git_helper/push/push_tree.rs b/v4_x/git-remote-gosh/src/git_helper/push/push_tree.rs index e094d0d5a..cee0cc0c2 100644 --- a/v4_x/git-remote-gosh/src/git_helper/push/push_tree.rs +++ b/v4_x/git-remote-gosh/src/git_helper/push/push_tree.rs @@ -152,15 +152,16 @@ pub async fn inner_deploy_tree( tree_address: &str, database: Arc, ) -> anyhow::Result<()> { - let tree = database.get_tree(tree_address)?; + let mut tree = database.get_tree(tree_address)?; tracing::trace!("inner_deploy_tree: remote_network={remote_network}, dao_addr={dao_addr}, remote_repo={remote_repo}, tree_id={}", tree.tree_id); let wallet = blockchain.user_wallet(&dao_addr, &remote_network).await?; blockchain .deploy_tree( &wallet, &tree.tree_id.to_hex().to_string(), + tree_address, &remote_repo, - &tree.tree_nodes, + &mut tree.tree_nodes, ) .await } diff --git a/v5_x/v5.0.0/git-remote-gosh/Cargo.toml b/v5_x/v5.0.0/git-remote-gosh/Cargo.toml index 8e43a7095..bd35c9ed0 100644 --- a/v5_x/v5.0.0/git-remote-gosh/Cargo.toml +++ b/v5_x/v5.0.0/git-remote-gosh/Cargo.toml @@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh'] name = 'git-remote-gosh' resolver = '2' rust-version = "1.68" -version = '5.1.27' +version = '5.1.29' [profile.profiling] debug = 1 diff --git a/v5_x/v5.0.0/git-remote-gosh/src/blockchain/service.rs b/v5_x/v5.0.0/git-remote-gosh/src/blockchain/service.rs index 4b310e54b..40769c6e0 100644 --- a/v5_x/v5.0.0/git-remote-gosh/src/blockchain/service.rs +++ b/v5_x/v5.0.0/git-remote-gosh/src/blockchain/service.rs @@ -213,8 +213,9 @@ pub mod tests { &self, wallet: &UserWallet, sha: &str, + tree_address: &str, repo_name: &str, - nodes: &HashMap, + nodes: &mut HashMap, ) -> anyhow::Result<()>; } diff --git a/v5_x/v5.0.0/git-remote-gosh/src/blockchain/tree/save.rs b/v5_x/v5.0.0/git-remote-gosh/src/blockchain/tree/save.rs index 5530395b2..5157376f8 100644 --- a/v5_x/v5.0.0/git-remote-gosh/src/blockchain/tree/save.rs +++ b/v5_x/v5.0.0/git-remote-gosh/src/blockchain/tree/save.rs @@ -1,8 +1,6 @@ -use crate::blockchain::contract::ContractInfo; -use crate::blockchain::user_wallet::UserWallet; use crate::blockchain::{ - call::BlockchainCall, BlockchainContractAddress, BlockchainService, Everscale, - GoshBlobBitFlags, Tree, + self, call::BlockchainCall, contract::ContractInfo, gosh_abi, user_wallet::UserWallet, + BlockchainContractAddress, BlockchainService, Everscale, GoshBlobBitFlags, GoshContract, Tree, }; use async_trait::async_trait; use git_object; @@ -59,12 +57,13 @@ pub trait DeployTree { &self, wallet: &UserWallet, sha: &str, + tree_address: &str, repo_name: &str, - nodes: &HashMap, + nodes: &mut HashMap, ) -> anyhow::Result<()>; } -static TREE_NODES_CHUNK_MAX_SIZE: usize = 200; +static TREE_NODES_CHUNK_MAX_SIZE: usize = 50; #[async_trait] impl DeployTree for Everscale { @@ -72,8 +71,9 @@ impl DeployTree for Everscale { &self, wallet: &UserWallet, sha: &str, + tree_address: &str, repo_name: &str, - nodes: &HashMap, // change to moved hashmap + nodes: &mut HashMap, // change to moved hashmap ) -> anyhow::Result<()> { let wallet_contract = wallet.take_one().await?; tracing::trace!("Acquired wallet: {}", wallet_contract.get_address()); @@ -86,21 +86,31 @@ impl DeployTree for Everscale { .await?; let mut nodes = nodes.to_owned(); let chunk: HashMap = HashMap::new(); - let params = DeployTreeArgs { - sha: sha.to_owned(), - repo_name: repo_name.to_owned(), - nodes: chunk, - number: nodes_cnt as u128, - }; - tracing::trace!("DeployTreeArgs: {params:?}"); - self.send_message( - wallet_contract.deref(), - "deployTree", - Some(serde_json::to_value(params.clone())?), - None, - ) - .await - .map(|_| ())?; + let tree_contract = + GoshContract::new(BlockchainContractAddress::new(&tree_address), gosh_abi::TREE); + + if tree_contract.is_active(self.client()).await? { + // check existing tree nodes + let onchain_tree_object = + blockchain::Tree::load(self.client(), &tree_address).await?; + nodes.retain(|k, _| !onchain_tree_object.objects.contains_key(k)); + } else { + let params = DeployTreeArgs { + sha: sha.to_owned(), + repo_name: repo_name.to_owned(), + nodes: chunk, + number: nodes_cnt as u128, + }; + tracing::trace!("DeployTreeArgs: {params:?}"); + self.send_message( + wallet_contract.deref(), + "deployTree", + Some(serde_json::to_value(params.clone())?), + None, + ) + .await + .map(|_| ())?; + } while nodes.len() > 0 { let mut counter = 0; let chunk: HashMap; diff --git a/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs b/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs index 8522fe3dc..2c5e2173f 100644 --- a/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs +++ b/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs @@ -1,8 +1,8 @@ -use crate::blockchain::contract::wait_contracts_deployed::wait_contracts_deployed; -use crate::blockchain::tree::load::check_if_tree_is_ready; use crate::{ blockchain::{ - tree::TreeNode, user_wallet::WalletError, AddrVersion, BlockchainContractAddress, + contract::wait_contracts_deployed::wait_contracts_deployed, + tree::{load::check_if_tree_is_ready, TreeNode}, + user_wallet::WalletError, AddrVersion, BlockchainContractAddress, BlockchainService, }, git_helper::{ @@ -330,8 +330,6 @@ impl ParallelTreeUploadSupport { let remote_network = context.remote.network.clone(); let repo = context.remote.repo.clone(); - tracing::trace!("Start push of tree: address: {tree_address:?}"); - self.expecting_deployed_contacts_addresses .push(tree_address.clone()); diff --git a/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/push_tree.rs b/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/push_tree.rs index e094d0d5a..cee0cc0c2 100644 --- a/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/push_tree.rs +++ b/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/push_tree.rs @@ -152,15 +152,16 @@ pub async fn inner_deploy_tree( tree_address: &str, database: Arc, ) -> anyhow::Result<()> { - let tree = database.get_tree(tree_address)?; + let mut tree = database.get_tree(tree_address)?; tracing::trace!("inner_deploy_tree: remote_network={remote_network}, dao_addr={dao_addr}, remote_repo={remote_repo}, tree_id={}", tree.tree_id); let wallet = blockchain.user_wallet(&dao_addr, &remote_network).await?; blockchain .deploy_tree( &wallet, &tree.tree_id.to_hex().to_string(), + tree_address, &remote_repo, - &tree.tree_nodes, + &mut tree.tree_nodes, ) .await } diff --git a/v5_x/v5.1.0/git-remote-gosh/Cargo.toml b/v5_x/v5.1.0/git-remote-gosh/Cargo.toml index 4862aab23..9f2c72b45 100644 --- a/v5_x/v5.1.0/git-remote-gosh/Cargo.toml +++ b/v5_x/v5.1.0/git-remote-gosh/Cargo.toml @@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh'] name = 'git-remote-gosh' resolver = '2' rust-version = "1.68" -version = '5.1.28' +version = '5.1.29' [profile.profiling] debug = 1 diff --git a/v5_x/v5.1.0/git-remote-gosh/src/blockchain/service.rs b/v5_x/v5.1.0/git-remote-gosh/src/blockchain/service.rs index 4b310e54b..40769c6e0 100644 --- a/v5_x/v5.1.0/git-remote-gosh/src/blockchain/service.rs +++ b/v5_x/v5.1.0/git-remote-gosh/src/blockchain/service.rs @@ -213,8 +213,9 @@ pub mod tests { &self, wallet: &UserWallet, sha: &str, + tree_address: &str, repo_name: &str, - nodes: &HashMap, + nodes: &mut HashMap, ) -> anyhow::Result<()>; } diff --git a/v5_x/v5.1.0/git-remote-gosh/src/blockchain/tree/save.rs b/v5_x/v5.1.0/git-remote-gosh/src/blockchain/tree/save.rs index 5530395b2..5157376f8 100644 --- a/v5_x/v5.1.0/git-remote-gosh/src/blockchain/tree/save.rs +++ b/v5_x/v5.1.0/git-remote-gosh/src/blockchain/tree/save.rs @@ -1,8 +1,6 @@ -use crate::blockchain::contract::ContractInfo; -use crate::blockchain::user_wallet::UserWallet; use crate::blockchain::{ - call::BlockchainCall, BlockchainContractAddress, BlockchainService, Everscale, - GoshBlobBitFlags, Tree, + self, call::BlockchainCall, contract::ContractInfo, gosh_abi, user_wallet::UserWallet, + BlockchainContractAddress, BlockchainService, Everscale, GoshBlobBitFlags, GoshContract, Tree, }; use async_trait::async_trait; use git_object; @@ -59,12 +57,13 @@ pub trait DeployTree { &self, wallet: &UserWallet, sha: &str, + tree_address: &str, repo_name: &str, - nodes: &HashMap, + nodes: &mut HashMap, ) -> anyhow::Result<()>; } -static TREE_NODES_CHUNK_MAX_SIZE: usize = 200; +static TREE_NODES_CHUNK_MAX_SIZE: usize = 50; #[async_trait] impl DeployTree for Everscale { @@ -72,8 +71,9 @@ impl DeployTree for Everscale { &self, wallet: &UserWallet, sha: &str, + tree_address: &str, repo_name: &str, - nodes: &HashMap, // change to moved hashmap + nodes: &mut HashMap, // change to moved hashmap ) -> anyhow::Result<()> { let wallet_contract = wallet.take_one().await?; tracing::trace!("Acquired wallet: {}", wallet_contract.get_address()); @@ -86,21 +86,31 @@ impl DeployTree for Everscale { .await?; let mut nodes = nodes.to_owned(); let chunk: HashMap = HashMap::new(); - let params = DeployTreeArgs { - sha: sha.to_owned(), - repo_name: repo_name.to_owned(), - nodes: chunk, - number: nodes_cnt as u128, - }; - tracing::trace!("DeployTreeArgs: {params:?}"); - self.send_message( - wallet_contract.deref(), - "deployTree", - Some(serde_json::to_value(params.clone())?), - None, - ) - .await - .map(|_| ())?; + let tree_contract = + GoshContract::new(BlockchainContractAddress::new(&tree_address), gosh_abi::TREE); + + if tree_contract.is_active(self.client()).await? { + // check existing tree nodes + let onchain_tree_object = + blockchain::Tree::load(self.client(), &tree_address).await?; + nodes.retain(|k, _| !onchain_tree_object.objects.contains_key(k)); + } else { + let params = DeployTreeArgs { + sha: sha.to_owned(), + repo_name: repo_name.to_owned(), + nodes: chunk, + number: nodes_cnt as u128, + }; + tracing::trace!("DeployTreeArgs: {params:?}"); + self.send_message( + wallet_contract.deref(), + "deployTree", + Some(serde_json::to_value(params.clone())?), + None, + ) + .await + .map(|_| ())?; + } while nodes.len() > 0 { let mut counter = 0; let chunk: HashMap; diff --git a/v5_x/v5.1.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs b/v5_x/v5.1.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs index 8522fe3dc..2c5e2173f 100644 --- a/v5_x/v5.1.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs +++ b/v5_x/v5.1.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs @@ -1,8 +1,8 @@ -use crate::blockchain::contract::wait_contracts_deployed::wait_contracts_deployed; -use crate::blockchain::tree::load::check_if_tree_is_ready; use crate::{ blockchain::{ - tree::TreeNode, user_wallet::WalletError, AddrVersion, BlockchainContractAddress, + contract::wait_contracts_deployed::wait_contracts_deployed, + tree::{load::check_if_tree_is_ready, TreeNode}, + user_wallet::WalletError, AddrVersion, BlockchainContractAddress, BlockchainService, }, git_helper::{ @@ -330,8 +330,6 @@ impl ParallelTreeUploadSupport { let remote_network = context.remote.network.clone(); let repo = context.remote.repo.clone(); - tracing::trace!("Start push of tree: address: {tree_address:?}"); - self.expecting_deployed_contacts_addresses .push(tree_address.clone()); diff --git a/v5_x/v5.1.0/git-remote-gosh/src/git_helper/push/push_tree.rs b/v5_x/v5.1.0/git-remote-gosh/src/git_helper/push/push_tree.rs index e094d0d5a..cee0cc0c2 100644 --- a/v5_x/v5.1.0/git-remote-gosh/src/git_helper/push/push_tree.rs +++ b/v5_x/v5.1.0/git-remote-gosh/src/git_helper/push/push_tree.rs @@ -152,15 +152,16 @@ pub async fn inner_deploy_tree( tree_address: &str, database: Arc, ) -> anyhow::Result<()> { - let tree = database.get_tree(tree_address)?; + let mut tree = database.get_tree(tree_address)?; tracing::trace!("inner_deploy_tree: remote_network={remote_network}, dao_addr={dao_addr}, remote_repo={remote_repo}, tree_id={}", tree.tree_id); let wallet = blockchain.user_wallet(&dao_addr, &remote_network).await?; blockchain .deploy_tree( &wallet, &tree.tree_id.to_hex().to_string(), + tree_address, &remote_repo, - &tree.tree_nodes, + &mut tree.tree_nodes, ) .await } diff --git a/version.json b/version.json index 8ed0234cb..e668b62db 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "version": "5.1.28" + "version": "5.1.29" }