From a33a101a952f5925b57233a93cd63f37e00c4260 Mon Sep 17 00:00:00 2001 From: Damir Vodenicarevic Date: Sat, 11 Jan 2025 23:37:17 +0100 Subject: [PATCH] update proto --- Cargo.lock | 202 +++++++++++++++--- Cargo.toml | 4 +- massa-execution-exports/src/mapping_grpc.rs | 8 +- massa-execution-worker/src/datastore_scan.rs | 32 ++- massa-execution-worker/src/execution.rs | 4 +- .../src/speculative_ledger.rs | 4 +- 6 files changed, 199 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 618705f818..a6a1002c70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -272,6 +272,34 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core 0.3.4", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 0.1.2", + "tower 0.4.13", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.7.9" @@ -279,7 +307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.5", "bytes", "futures-util", "http 1.2.0", @@ -293,12 +321,29 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "sync_wrapper", + "sync_wrapper 1.0.2", "tower 0.5.2", "tower-layer", "tower-service", ] +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.4.5" @@ -314,7 +359,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", ] @@ -2037,6 +2082,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -2089,6 +2135,18 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.32", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-timeout" version = "0.5.2" @@ -2915,21 +2973,20 @@ dependencies = [ [[package]] name = "massa-proto-rs" version = "0.1.0" -source = "git+https://github.com/massalabs/massa-proto-rs?rev=ebddff4d1b1e29dec9255fd8cae8151617ac68e9#ebddff4d1b1e29dec9255fd8cae8151617ac68e9" +source = "git+https://github.com/massalabs/massa-proto-rs?rev=1ab9a65e7b9f0077361cafe905c2a26ba04ae9d1#1ab9a65e7b9f0077361cafe905c2a26ba04ae9d1" dependencies = [ "glob", - "prost", + "prost 0.12.6", "prost-build", - "prost-types", - "tonic", + "prost-types 0.12.6", + "tonic 0.10.2", "tonic-build", - "tower-service", ] [[package]] name = "massa-sc-runtime" version = "0.10.0" -source = "git+https://github.com/massalabs/massa-sc-runtime?rev=ddffc7d8443b9844b05453fe8f4c8d4ccb5b5bf4#ddffc7d8443b9844b05453fe8f4c8d4ccb5b5bf4" +source = "git+https://github.com/massalabs/massa-sc-runtime?rev=8013e743d7b3efe46cfc760b05337512d7f10073#8013e743d7b3efe46cfc760b05337512d7f10073" dependencies = [ "anyhow", "as-ffi-bindings", @@ -2942,8 +2999,8 @@ dependencies = [ "more-asserts 0.3.1", "num_enum", "parking_lot", - "prost", - "prost-types", + "prost 0.13.4", + "prost-types 0.13.4", "rand", "regex", "rust_decimal", @@ -3309,7 +3366,7 @@ dependencies = [ "mockall", "num", "parking_lot", - "prost", + "prost 0.13.4", "rand", "rand_xoshiro", "rocksdb", @@ -3430,7 +3487,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", - "tonic", + "tonic 0.12.3", "tonic-health", "tonic-reflection", "tonic-web", @@ -3709,7 +3766,7 @@ dependencies = [ "massa_time", "rcgen", "thiserror 1.0.69", - "tonic", + "tonic 0.12.3", "tracing", ] @@ -4628,6 +4685,16 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive 0.12.6", +] + [[package]] name = "prost" version = "0.13.4" @@ -4635,29 +4702,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.4", ] [[package]] name = "prost-build" -version = "0.13.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ - "heck 0.5.0", - "itertools 0.13.0", + "bytes", + "heck 0.4.1", + "itertools 0.10.5", "log", "multimap", "once_cell", "petgraph", "prettyplease", - "prost", - "prost-types", + "prost 0.12.6", + "prost-types 0.12.6", "regex", "syn 2.0.95", "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2 1.0.92", + "quote 1.0.38", + "syn 2.0.95", +] + [[package]] name = "prost-derive" version = "0.13.4" @@ -4671,13 +4752,22 @@ dependencies = [ "syn 2.0.95", ] +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost 0.12.6", +] + [[package]] name = "prost-types" version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" dependencies = [ - "prost", + "prost 0.13.4", ] [[package]] @@ -5753,6 +5843,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5969,6 +6065,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.4.0" @@ -6070,6 +6176,33 @@ dependencies = [ "winnow 0.6.22", ] +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.6.20", + "base64 0.21.7", + "bytes", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-timeout 0.4.1", + "percent-encoding", + "pin-project", + "prost 0.12.6", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic" version = "0.12.3" @@ -6078,7 +6211,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.7.9", "base64 0.22.1", "bytes", "flate2", @@ -6087,11 +6220,11 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.5.2", - "hyper-timeout", + "hyper-timeout 0.5.2", "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.4", "rustls-pemfile", "socket2", "tokio", @@ -6105,14 +6238,13 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.12.3" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" dependencies = [ "prettyplease", "proc-macro2 1.0.92", "prost-build", - "prost-types", "quote 1.0.38", "syn 2.0.95", ] @@ -6124,10 +6256,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eaf34ddb812120f5c601162d5429933c9b527d901ab0e7f930d3147e33a09b2" dependencies = [ "async-stream", - "prost", + "prost 0.13.4", "tokio", "tokio-stream", - "tonic", + "tonic 0.12.3", ] [[package]] @@ -6136,11 +6268,11 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "878d81f52e7fcfd80026b7fdb6a9b578b3c3653ba987f87f0dce4b64043cba27" dependencies = [ - "prost", - "prost-types", + "prost 0.13.4", + "prost-types 0.13.4", "tokio", "tokio-stream", - "tonic", + "tonic 0.12.3", ] [[package]] @@ -6156,7 +6288,7 @@ dependencies = [ "http-body-util", "pin-project", "tokio-stream", - "tonic", + "tonic 0.12.3", "tower-http 0.5.2", "tower-layer", "tower-service", @@ -6193,7 +6325,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", ] diff --git a/Cargo.toml b/Cargo.toml index 35b6fca800..88fbc999bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,8 +109,8 @@ massa_wallet = { path = "./massa-wallet" } massa_event_cache = { path = "./massa-event-cache" } # Massa projects dependencies -massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", "rev" = "ebddff4d1b1e29dec9255fd8cae8151617ac68e9" } -massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "ddffc7d8443b9844b05453fe8f4c8d4ccb5b5bf4" } +massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", "rev" = "1ab9a65e7b9f0077361cafe905c2a26ba04ae9d1" } +massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "8013e743d7b3efe46cfc760b05337512d7f10073" } peernet = { git = "https://github.com/massalabs/PeerNet", "rev" = "04b05ddd320fbe76cc858115af7b5fc28bdb8310" } diff --git a/massa-execution-exports/src/mapping_grpc.rs b/massa-execution-exports/src/mapping_grpc.rs index c928f8df53..cea90da32e 100644 --- a/massa-execution-exports/src/mapping_grpc.rs +++ b/massa-execution-exports/src/mapping_grpc.rs @@ -68,7 +68,7 @@ pub fn to_querystate_filter( let start_key = match (value.start_key, value.inclusive_start_key) { (None, _) => std::ops::Bound::Unbounded, - (Some(k), inclusive) => { + (Some(mut k), inclusive) => { if k.len() > max_datastore_key_length as usize { // the key is longer than the max possible length // it will be by definition excluded @@ -85,7 +85,7 @@ pub fn to_querystate_filter( let end_key = match (value.end_key, value.inclusive_end_key) { (None, _) => std::ops::Bound::Unbounded, - (Some(k), inclusive) => { + (Some(mut k), inclusive) => { if k.len() > max_datastore_key_length as usize { // the key is longer than the max possible length // it will be by definition excluded @@ -123,7 +123,7 @@ pub fn to_querystate_filter( let start_key = match (value.start_key, value.inclusive_start_key) { (None, _) => std::ops::Bound::Unbounded, - (Some(k), inclusive) => { + (Some(mut k), inclusive) => { if k.len() > max_datastore_key_length as usize { // the key is longer than the max possible length // it will be by definition excluded @@ -140,7 +140,7 @@ pub fn to_querystate_filter( let end_key = match (value.end_key, value.inclusive_end_key) { (None, _) => std::ops::Bound::Unbounded, - (Some(k), inclusive) => { + (Some(mut k), inclusive) => { if k.len() > max_datastore_key_length as usize { // the key is longer than the max possible length // it will be by definition excluded diff --git a/massa-execution-worker/src/datastore_scan.rs b/massa-execution-worker/src/datastore_scan.rs index 48c55071bb..ec66154135 100644 --- a/massa-execution-worker/src/datastore_scan.rs +++ b/massa-execution-worker/src/datastore_scan.rs @@ -1,13 +1,22 @@ //! A file describing an optimized datastore keys traversal algorithm. //! It is shared between execution.rs and speculative_ledger.rs. -use std::{collections::BTreeSet, ops::Bound}; +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + ops::Bound, + sync::Arc, +}; +use massa_final_state::{FinalState, FinalStateController}; +use massa_ledger_exports::LedgerChanges; use massa_models::{ address::Address, datastore::{get_prefix_bounds, range_intersection}, - ledger::LedgerChange, + types::{SetOrDelete, SetUpdateOrDelete}, }; +use parking_lot::RwLock; + +use crate::active_history::ActiveHistory; /// Gets a copy of a datastore keys for a given address /// @@ -28,9 +37,9 @@ pub fn scan_datastore( start_key: Bound>, end_key: Bound>, count: Option, - final_state: &Arc>, - active_history: &Arc>, - added_changes: Option<&LedgerChange>, + final_state: Arc>, + active_history: Arc>, + added_changes: Option<&LedgerChanges>, ) -> (Option>>, Option>>) { // get final keys let mut final_keys = final_state.read().get_ledger().get_datastore_keys( @@ -55,17 +64,18 @@ pub fn scan_datastore( let mut key_updates = BTreeMap::new(); { let mut update_indices = VecDeque::new(); - let history_lock = history_lock.active_history.read(); + let history_lock = active_history.read(); let mut it = history_lock .0 .iter() .map(|v| v.state_changes.ledger_changes); if let Some(ac) = added_changes.as_ref() { + // if there are added changes, chain them as the last index it = it.chain(std::iter::once(*ac)); } for (index, output) in it.enumerate().rev() { - match output.state_changes.ledger_changes.get(addr) { + match output.get(addr) { // address absent from the changes None => (), @@ -107,8 +117,10 @@ pub fn scan_datastore( for idx in update_indices { let changes = if idx < history_lock.0.len() { &history_lock.0[idx].state_changes.ledger_changes + } else if let Some(added_changes) = added_changes.as_ref() { + *added_changes } else { - added_changes.expect("unexpected index out of bounds") + panic!("unexpected index out of bounds") }; if let SetUpdateOrDelete::Update(updates) = changes @@ -245,8 +257,8 @@ pub fn scan_datastore( } } - if let Some(last_k) = last_final_batch_key.as_ref() { - if final_keys_queue.is_empty() { + if final_keys_queue.is_empty() { + if let Some(last_k) = last_final_batch_key { // the last final item was consumed: replenish the queue by querying more final_keys_queue = final_state .read() diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index b112665d8e..4b9cebb3c3 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -2222,8 +2222,8 @@ impl ExecutionState { start_key, end_key, count, - &self.final_state, - &self.active_history, + self.final_state.clone(), + self.active_history.clone(), None, ) } diff --git a/massa-execution-worker/src/speculative_ledger.rs b/massa-execution-worker/src/speculative_ledger.rs index 3709591357..2e18653781 100644 --- a/massa-execution-worker/src/speculative_ledger.rs +++ b/massa-execution-worker/src/speculative_ledger.rs @@ -378,8 +378,8 @@ impl SpeculativeLedger { start_key, end_key, count, - &self.final_state, - &self.active_history, + self.final_state.clone(), + self.active_history.clone(), Some(&self.added_changes), ) .1