Fixed uploading for big trees (#528)

* fixed uploading for big trees * bump up version
gosh-sh · Aug 1, 2023 · 5e7019a · 5e7019a
1 parent e741f35
commit 5e7019a
Show file tree

Hide file tree

Showing 16 changed files with 124 additions and 94 deletions.
diff --git a/v4_x/git-remote-gosh/Cargo.toml b/v4_x/git-remote-gosh/Cargo.toml
@@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh']
 name = 'git-remote-gosh'
 resolver = '2'
 rust-version = "1.68"
-version = '4.1.26'
+version = '4.1.29'
 
 [profile.profiling]
 debug = 1

diff --git a/v4_x/git-remote-gosh/src/blockchain/service.rs b/v4_x/git-remote-gosh/src/blockchain/service.rs
@@ -213,8 +213,9 @@ pub mod tests {
                 &self,
                 wallet: &UserWallet,
                 sha: &str,
+                tree_address: &str,
                 repo_name: &str,
-                nodes: &HashMap<String, TreeNode>,
+                nodes: &mut HashMap<String, TreeNode>,
             ) -> anyhow::Result<()>;
         }
 

diff --git a/v4_x/git-remote-gosh/src/blockchain/tree/save.rs b/v4_x/git-remote-gosh/src/blockchain/tree/save.rs
@@ -1,8 +1,6 @@
-use crate::blockchain::contract::ContractInfo;
-use crate::blockchain::user_wallet::UserWallet;
 use crate::blockchain::{
-    call::BlockchainCall, BlockchainContractAddress, BlockchainService, Everscale,
-    GoshBlobBitFlags, Tree,
+    self, call::BlockchainCall, contract::ContractInfo, gosh_abi, user_wallet::UserWallet,
+    BlockchainContractAddress, BlockchainService, GoshContract, Everscale, GoshBlobBitFlags, Tree,
 };
 use async_trait::async_trait;
 use git_object;
@@ -59,21 +57,23 @@ pub trait DeployTree {
         &self,
         wallet: &UserWallet,
         sha: &str,
+        tree_address: &str,
         repo_name: &str,
-        nodes: &HashMap<String, TreeNode>,
+        nodes: &mut HashMap<String, TreeNode>,
     ) -> anyhow::Result<()>;
 }
 
-static TREE_NODES_CHUNK_MAX_SIZE: usize = 200;
+static TREE_NODES_CHUNK_MAX_SIZE: usize = 50;
 
 #[async_trait]
 impl DeployTree for Everscale {
     async fn deploy_tree(
         &self,
         wallet: &UserWallet,
         sha: &str,
+        tree_address: &str,
         repo_name: &str,
-        nodes: &HashMap<String, TreeNode>, // change to moved hashmap
+        nodes: &mut HashMap<String, TreeNode>, // change to moved hashmap
     ) -> anyhow::Result<()> {
         let wallet_contract = wallet.take_one().await?;
         tracing::trace!("Acquired wallet: {}", wallet_contract.get_address());
@@ -86,21 +86,30 @@ impl DeployTree for Everscale {
                     .await?;
             let mut nodes = nodes.to_owned();
             let chunk: HashMap<String, TreeNode> = HashMap::new();
-            let params = DeployTreeArgs {
-                sha: sha.to_owned(),
-                repo_name: repo_name.to_owned(),
-                nodes: chunk,
-                number: nodes_cnt as u128,
-            };
-            tracing::trace!("DeployTreeArgs: {params:?}");
-            self.send_message(
-                wallet_contract.deref(),
-                "deployTree",
-                Some(serde_json::to_value(params.clone())?),
-                None,
-            )
-            .await
-            .map(|_| ())?;
+            let tree_contract =
+                GoshContract::new(BlockchainContractAddress::new(&tree_address), gosh_abi::TREE);
+
+            if tree_contract.is_active(self.client()).await? {
+                let onchain_tree_object =
+                    blockchain::Tree::load(self.client(), &tree_address).await?;
+                nodes.retain(|k, _| !onchain_tree_object.objects.contains_key(k));
+            } else {
+                let params = DeployTreeArgs {
+                    sha: sha.to_owned(),
+                    repo_name: repo_name.to_owned(),
+                    nodes: chunk,
+                    number: nodes_cnt as u128,
+                };
+                tracing::trace!("DeployTreeArgs: {params:?}");
+                self.send_message(
+                    wallet_contract.deref(),
+                    "deployTree",
+                    Some(serde_json::to_value(params.clone())?),
+                    None,
+                )
+                .await
+                .map(|_| ())?;
+            }
             while nodes.len() > 0 {
                 let mut counter = 0;
                 let chunk: HashMap<String, TreeNode>;

diff --git a/v4_x/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs b/v4_x/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs
@@ -1,8 +1,9 @@
-use crate::blockchain::contract::wait_contracts_deployed::wait_contracts_deployed;
-use crate::blockchain::tree::load::check_if_tree_is_ready;
 use crate::{
     blockchain::{
-        tree::TreeNode, user_wallet::WalletError, AddrVersion, BlockchainContractAddress,
+        contract::{wait_contracts_deployed::wait_contracts_deployed, GoshContract},
+        gosh_abi,
+        tree::{load::check_if_tree_is_ready, TreeNode},
+        user_wallet::WalletError, AddrVersion, BlockchainContractAddress,
         BlockchainService,
     },
     git_helper::{
@@ -330,8 +331,6 @@ impl ParallelTreeUploadSupport {
         let remote_network = context.remote.network.clone();
         let repo = context.remote.repo.clone();
 
-        tracing::trace!("Start push of tree: address: {tree_address:?}");
-
         self.expecting_deployed_contacts_addresses
             .push(tree_address.clone());
 

diff --git a/v4_x/git-remote-gosh/src/git_helper/push/push_tree.rs b/v4_x/git-remote-gosh/src/git_helper/push/push_tree.rs
@@ -152,15 +152,16 @@ pub async fn inner_deploy_tree(
     tree_address: &str,
     database: Arc<GoshDB>,
 ) -> anyhow::Result<()> {
-    let tree = database.get_tree(tree_address)?;
+    let mut tree = database.get_tree(tree_address)?;
     tracing::trace!("inner_deploy_tree: remote_network={remote_network}, dao_addr={dao_addr}, remote_repo={remote_repo}, tree_id={}", tree.tree_id);
     let wallet = blockchain.user_wallet(&dao_addr, &remote_network).await?;
     blockchain
         .deploy_tree(
             &wallet,
             &tree.tree_id.to_hex().to_string(),
+            tree_address,
             &remote_repo,
-            &tree.tree_nodes,
+            &mut tree.tree_nodes,
         )
         .await
 }
diff --git a/v5_x/v5.0.0/git-remote-gosh/Cargo.toml b/v5_x/v5.0.0/git-remote-gosh/Cargo.toml
@@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh']
 name = 'git-remote-gosh'
 resolver = '2'
 rust-version = "1.68"
-version = '5.1.27'
+version = '5.1.29'
 
 [profile.profiling]
 debug = 1

diff --git a/v5_x/v5.0.0/git-remote-gosh/src/blockchain/service.rs b/v5_x/v5.0.0/git-remote-gosh/src/blockchain/service.rs
@@ -213,8 +213,9 @@ pub mod tests {
                 &self,
                 wallet: &UserWallet,
                 sha: &str,
+                tree_address: &str,
                 repo_name: &str,
-                nodes: &HashMap<String, TreeNode>,
+                nodes: &mut HashMap<String, TreeNode>,
             ) -> anyhow::Result<()>;
         }
 

diff --git a/v5_x/v5.0.0/git-remote-gosh/src/blockchain/tree/save.rs b/v5_x/v5.0.0/git-remote-gosh/src/blockchain/tree/save.rs
@@ -1,8 +1,6 @@
-use crate::blockchain::contract::ContractInfo;
-use crate::blockchain::user_wallet::UserWallet;
 use crate::blockchain::{
-    call::BlockchainCall, BlockchainContractAddress, BlockchainService, Everscale,
-    GoshBlobBitFlags, Tree,
+    self, call::BlockchainCall, contract::ContractInfo, gosh_abi, user_wallet::UserWallet,
+    BlockchainContractAddress, BlockchainService, Everscale, GoshBlobBitFlags, GoshContract, Tree,
 };
 use async_trait::async_trait;
 use git_object;
@@ -59,21 +57,23 @@ pub trait DeployTree {
         &self,
         wallet: &UserWallet,
         sha: &str,
+        tree_address: &str,
         repo_name: &str,
-        nodes: &HashMap<String, TreeNode>,
+        nodes: &mut HashMap<String, TreeNode>,
     ) -> anyhow::Result<()>;
 }
 
-static TREE_NODES_CHUNK_MAX_SIZE: usize = 200;
+static TREE_NODES_CHUNK_MAX_SIZE: usize = 50;
 
 #[async_trait]
 impl DeployTree for Everscale {
     async fn deploy_tree(
         &self,
         wallet: &UserWallet,
         sha: &str,
+        tree_address: &str,
         repo_name: &str,
-        nodes: &HashMap<String, TreeNode>, // change to moved hashmap
+        nodes: &mut HashMap<String, TreeNode>, // change to moved hashmap
     ) -> anyhow::Result<()> {
         let wallet_contract = wallet.take_one().await?;
         tracing::trace!("Acquired wallet: {}", wallet_contract.get_address());
@@ -86,21 +86,31 @@ impl DeployTree for Everscale {
                     .await?;
             let mut nodes = nodes.to_owned();
             let chunk: HashMap<String, TreeNode> = HashMap::new();
-            let params = DeployTreeArgs {
-                sha: sha.to_owned(),
-                repo_name: repo_name.to_owned(),
-                nodes: chunk,
-                number: nodes_cnt as u128,
-            };
-            tracing::trace!("DeployTreeArgs: {params:?}");
-            self.send_message(
-                wallet_contract.deref(),
-                "deployTree",
-                Some(serde_json::to_value(params.clone())?),
-                None,
-            )
-            .await
-            .map(|_| ())?;
+            let tree_contract =
+                GoshContract::new(BlockchainContractAddress::new(&tree_address), gosh_abi::TREE);
+
+            if tree_contract.is_active(self.client()).await? {
+                // check existing tree nodes
+                let onchain_tree_object =
+                    blockchain::Tree::load(self.client(), &tree_address).await?;
+                nodes.retain(|k, _| !onchain_tree_object.objects.contains_key(k));
+            } else {
+                let params = DeployTreeArgs {
+                    sha: sha.to_owned(),
+                    repo_name: repo_name.to_owned(),
+                    nodes: chunk,
+                    number: nodes_cnt as u128,
+                };
+                tracing::trace!("DeployTreeArgs: {params:?}");
+                self.send_message(
+                    wallet_contract.deref(),
+                    "deployTree",
+                    Some(serde_json::to_value(params.clone())?),
+                    None,
+                )
+                .await
+                .map(|_| ())?;
+            }
             while nodes.len() > 0 {
                 let mut counter = 0;
                 let chunk: HashMap<String, TreeNode>;

diff --git a/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs b/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/parallel_snapshot_upload_support.rs
@@ -1,8 +1,8 @@
-use crate::blockchain::contract::wait_contracts_deployed::wait_contracts_deployed;
-use crate::blockchain::tree::load::check_if_tree_is_ready;
 use crate::{
     blockchain::{
-        tree::TreeNode, user_wallet::WalletError, AddrVersion, BlockchainContractAddress,
+        contract::wait_contracts_deployed::wait_contracts_deployed,
+        tree::{load::check_if_tree_is_ready, TreeNode},
+        user_wallet::WalletError, AddrVersion, BlockchainContractAddress,
         BlockchainService,
     },
     git_helper::{
@@ -330,8 +330,6 @@ impl ParallelTreeUploadSupport {
         let remote_network = context.remote.network.clone();
         let repo = context.remote.repo.clone();
 
-        tracing::trace!("Start push of tree: address: {tree_address:?}");
-
         self.expecting_deployed_contacts_addresses
             .push(tree_address.clone());
 

diff --git a/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/push_tree.rs b/v5_x/v5.0.0/git-remote-gosh/src/git_helper/push/push_tree.rs
@@ -152,15 +152,16 @@ pub async fn inner_deploy_tree(
     tree_address: &str,
     database: Arc<GoshDB>,
 ) -> anyhow::Result<()> {
-    let tree = database.get_tree(tree_address)?;
+    let mut tree = database.get_tree(tree_address)?;
     tracing::trace!("inner_deploy_tree: remote_network={remote_network}, dao_addr={dao_addr}, remote_repo={remote_repo}, tree_id={}", tree.tree_id);
     let wallet = blockchain.user_wallet(&dao_addr, &remote_network).await?;
     blockchain
         .deploy_tree(
             &wallet,
             &tree.tree_id.to_hex().to_string(),
+            tree_address,
             &remote_repo,
-            &tree.tree_nodes,
+            &mut tree.tree_nodes,
         )
         .await
 }
diff --git a/v5_x/v5.1.0/git-remote-gosh/Cargo.toml b/v5_x/v5.1.0/git-remote-gosh/Cargo.toml
@@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh']
 name = 'git-remote-gosh'
 resolver = '2'
 rust-version = "1.68"
-version = '5.1.28'
+version = '5.1.29'
 
 [profile.profiling]
 debug = 1

diff --git a/v5_x/v5.1.0/git-remote-gosh/src/blockchain/service.rs b/v5_x/v5.1.0/git-remote-gosh/src/blockchain/service.rs
@@ -213,8 +213,9 @@ pub mod tests {
                 &self,
                 wallet: &UserWallet,
                 sha: &str,
+                tree_address: &str,
                 repo_name: &str,
-                nodes: &HashMap<String, TreeNode>,
+                nodes: &mut HashMap<String, TreeNode>,
             ) -> anyhow::Result<()>;
         }
 

diff --git a/v5_x/v5.1.0/git-remote-gosh/src/blockchain/tree/save.rs b/v5_x/v5.1.0/git-remote-gosh/src/blockchain/tree/save.rs
@@ -1,8 +1,6 @@
-use crate::blockchain::contract::ContractInfo;
-use crate::blockchain::user_wallet::UserWallet;
 use crate::blockchain::{
-    call::BlockchainCall, BlockchainContractAddress, BlockchainService, Everscale,
-    GoshBlobBitFlags, Tree,
+    self, call::BlockchainCall, contract::ContractInfo, gosh_abi, user_wallet::UserWallet,
+    BlockchainContractAddress, BlockchainService, Everscale, GoshBlobBitFlags, GoshContract, Tree,
 };
 use async_trait::async_trait;
 use git_object;
@@ -59,21 +57,23 @@ pub trait DeployTree {
         &self,
         wallet: &UserWallet,
         sha: &str,
+        tree_address: &str,
         repo_name: &str,
-        nodes: &HashMap<String, TreeNode>,
+        nodes: &mut HashMap<String, TreeNode>,
     ) -> anyhow::Result<()>;
 }
 
-static TREE_NODES_CHUNK_MAX_SIZE: usize = 200;
+static TREE_NODES_CHUNK_MAX_SIZE: usize = 50;
 
 #[async_trait]
 impl DeployTree for Everscale {
     async fn deploy_tree(
         &self,
         wallet: &UserWallet,
         sha: &str,
+        tree_address: &str,
         repo_name: &str,
-        nodes: &HashMap<String, TreeNode>, // change to moved hashmap
+        nodes: &mut HashMap<String, TreeNode>, // change to moved hashmap
     ) -> anyhow::Result<()> {
         let wallet_contract = wallet.take_one().await?;
         tracing::trace!("Acquired wallet: {}", wallet_contract.get_address());
@@ -86,21 +86,31 @@ impl DeployTree for Everscale {
                     .await?;
             let mut nodes = nodes.to_owned();
             let chunk: HashMap<String, TreeNode> = HashMap::new();
-            let params = DeployTreeArgs {
-                sha: sha.to_owned(),
-                repo_name: repo_name.to_owned(),
-                nodes: chunk,
-                number: nodes_cnt as u128,
-            };
-            tracing::trace!("DeployTreeArgs: {params:?}");
-            self.send_message(
-                wallet_contract.deref(),
-                "deployTree",
-                Some(serde_json::to_value(params.clone())?),
-                None,
-            )
-            .await
-            .map(|_| ())?;
+            let tree_contract =
+                GoshContract::new(BlockchainContractAddress::new(&tree_address), gosh_abi::TREE);
+
+            if tree_contract.is_active(self.client()).await? {
+                // check existing tree nodes
+                let onchain_tree_object =
+                    blockchain::Tree::load(self.client(), &tree_address).await?;
+                nodes.retain(|k, _| !onchain_tree_object.objects.contains_key(k));
+            } else {
+                let params = DeployTreeArgs {
+                    sha: sha.to_owned(),
+                    repo_name: repo_name.to_owned(),
+                    nodes: chunk,
+                    number: nodes_cnt as u128,
+                };
+                tracing::trace!("DeployTreeArgs: {params:?}");
+                self.send_message(
+                    wallet_contract.deref(),
+                    "deployTree",
+                    Some(serde_json::to_value(params.clone())?),
+                    None,
+                )
+                .await
+                .map(|_| ())?;
+            }
             while nodes.len() > 0 {
                 let mut counter = 0;
                 let chunk: HashMap<String, TreeNode>;