From 6daa5cfa2bc420a4bb6e911586957e3918911dff Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 17:21:50 +0100 Subject: [PATCH 01/23] essential --- .github/workflows/lint.yml | 8 + .github/workflows/test.yml | 4 + CHANGELOG.md | 1 + Cargo.toml | 2 +- crates/store/Cargo.toml | 11 +- crates/store/README.md | 29 +++ .../store/benches/account_tree_historical.rs | 238 ------------------ crates/store/src/accounts/mod.rs | 4 + crates/store/src/lib.rs | 2 + crates/store/src/state.rs | 48 +++- 10 files changed, 99 insertions(+), 248 deletions(-) delete mode 100644 crates/store/benches/account_tree_historical.rs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8cebcbe97..a248394d9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -51,6 +51,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@main + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: | rustup update --no-self-update @@ -87,6 +91,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@main + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cfee5fc3c..f718d65ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,10 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@main + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index f0c115da2..602ca3520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,7 @@ - Add optional `TransactionInputs` field to `SubmitProvenTransaction` endpoint for transaction re-execution (#[1278](https://github.com/0xMiden/miden-node/pull/1278)). - Added `validator` crate with initial protobuf, gRPC server, and sub-command (#[1293](https://github.com/0xMiden/miden-node/pull/1293)). - [BREAKING] Added `AccountTreeWithHistory` and integrate historical queries into `GetAccountProof` ([#1292](https://github.com/0xMiden/miden-node/pull/1292)). +- [BREAKING] Added `rocksdb` feature to enable rocksdb backends of `LargeSmt` ([#1326](https://github.com/0xMiden/miden-node/pull/1326)). - [BREAKING] Handle past/historical `AccountProof` requests ([#1333](https://github.com/0xMiden/miden-node/pull/1333)). - Implement `DataStore::get_note_script()` for `NtxDataStore` (#[1332](https://github.com/0xMiden/miden-node/pull/1332)). - Started validating notes by their commitment instead of ID before entering the mempool ([#1338](https://github.com/0xMiden/miden-node/pull/1338)). diff --git a/Cargo.toml b/Cargo.toml index 53e5182bb..c1cf8f3da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ miden-tx = { branch = "next", default-features = false, git = "http miden-tx-batch-prover = { branch = "next", git = "https://github.com/0xMiden/miden-base.git" } # Other miden dependencies. These should align with those expected by miden-base. -miden-air = { features = ["std", "testing"], version = "0.20" } +miden-air = { features = ["std", "testing"], version = "0.20" } # External dependencies anyhow = { version = "1.0" } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 22037e4b9..ff737ffe7 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -24,6 +24,7 @@ diesel_migrations = { features = ["sqlite"], version = "2.3" } fs-err = { workspace = true } hex = { version = "0.4" } indexmap = { workspace = true } +miden-crypto = { features = ["concurrent", "hashmaps"], workspace = true } miden-node-proto = { workspace = true } miden-node-proto-build = { features = ["internal"], workspace = true } miden-node-utils = { workspace = true } @@ -55,6 +56,14 @@ rand = { workspace = true } regex = { version = "1.11" } termtree = { version = "0.5" } +[features] +default = ["rocksdb"] +rocksdb = ["miden-crypto/rocksdb"] + [[bench]] harness = false -name = "account_tree_historical" +name = "account_tree" +required-features = ["rocksdb"] + +[package.metadata.cargo-machete] +ignored = ["miden-crypto"] diff --git a/crates/store/README.md b/crates/store/README.md index e3f9a8dde..57c002fe5 100644 --- a/crates/store/README.md +++ b/crates/store/README.md @@ -7,6 +7,35 @@ operator must take care that the store's API endpoint is **only** exposed to the For more information on the installation and operation of this component, please see the [node's readme](/README.md). +## RocksDB Feature + +The `rocksdb` feature (enabled by default) provides disk-backed storage via RocksDB for `LargeSmt`. Building _requires_ LLVM/Clang for `bindgen`. + +### Using System Libraries + +To avoid compiling RocksDB from source and safe yourself some time, use system libraries: + +```bash +# Install system RocksDB +# (Ubuntu/Debian) +#sudo apt-get install librocksdb-dev clang llvm-dev libclang-dev +# (Fedora) +#sudo dnf install rocksdb rocksdb-devel llvm19 clang19 + +# Set environment variables to use system library +export ROCKSDB_LIB_DIR=/usr/lib +export ROCKSDB_INCLUDE_DIR=/usr/include +# export ROCKSDB_STATIC=1 (optional) +# (Ubuntu/Debian) +#export LIBCLANG_PATH=/usr/lib/llvm-14/lib +# (Fedora) +#export LIBCLANG_PATH=/usr/lib64/llvm19/lib +``` + +### Building from Source + +Without the environment variables above, `librocksdb-sys` compiles RocksDB from source, which requires a C/C++ toolchain. + ## API overview The full gRPC API can be found [here](../../proto/proto/store.proto). diff --git a/crates/store/benches/account_tree_historical.rs b/crates/store/benches/account_tree_historical.rs deleted file mode 100644 index ba7a5c2cc..000000000 --- a/crates/store/benches/account_tree_historical.rs +++ /dev/null @@ -1,238 +0,0 @@ -use std::hint::black_box; - -use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use miden_node_store::AccountTreeWithHistory; -use miden_protocol::Word; -use miden_protocol::account::AccountId; -use miden_protocol::block::BlockNumber; -use miden_protocol::block::account_tree::{AccountTree, account_id_to_smt_key}; -use miden_protocol::crypto::hash::rpo::Rpo256; -use miden_protocol::crypto::merkle::smt::{LargeSmt, MemoryStorage}; -use miden_protocol::testing::account_id::AccountIdBuilder; - -// HELPER FUNCTIONS -// ================================================================================================ - -/// Creates a storage backend for a `LargeSmt`. -fn setup_storage() -> MemoryStorage { - // TODO migrate to RocksDB for persistence to gain meaningful numbers - MemoryStorage::default() -} - -/// Generates a deterministic word from a seed. -fn generate_word(seed: &mut [u8; 32]) -> Word { - for byte in seed.iter_mut() { - *byte = byte.wrapping_add(1); - if *byte != 0 { - break; - } - } - Rpo256::hash(seed) -} - -/// Generates a deterministic `AccountId` from a seed. -fn generate_account_id(seed: &mut [u8; 32]) -> AccountId { - for byte in seed.iter_mut() { - *byte = byte.wrapping_add(1); - if *byte != 0 { - break; - } - } - AccountIdBuilder::new().build_with_seed(*seed) -} - -// SETUP FUNCTIONS -// ================================================================================================ - -/// Sets up a vanilla `AccountTree` with specified number of accounts. -fn setup_vanilla_account_tree( - num_accounts: usize, -) -> (AccountTree>, Vec) { - let mut seed = [0u8; 32]; - let mut account_ids = Vec::new(); - let mut entries = Vec::new(); - - for _ in 0..num_accounts { - let account_id = generate_account_id(&mut seed); - let commitment = generate_word(&mut seed); - account_ids.push(account_id); - entries.push((account_id_to_smt_key(account_id), commitment)); - } - - let storage = setup_storage(); - let smt = - LargeSmt::with_entries(storage, entries).expect("Failed to create LargeSmt from entries"); - let tree = AccountTree::new(smt).expect("Failed to create AccountTree"); - (tree, account_ids) -} - -/// Sets up `AccountTreeWithHistory` with specified number of accounts and blocks. -fn setup_account_tree_with_history( - num_accounts: usize, - num_blocks: usize, -) -> (AccountTreeWithHistory, Vec) { - let mut seed = [0u8; 32]; - let storage = setup_storage(); - let smt = LargeSmt::with_entries(storage, std::iter::empty()) - .expect("Failed to create empty LargeSmt"); - let account_tree = AccountTree::new(smt).expect("Failed to create AccountTree"); - let mut account_tree_hist = AccountTreeWithHistory::new(account_tree, BlockNumber::GENESIS); - let mut account_ids = Vec::new(); - - for _block in 0..num_blocks { - let mutations: Vec<_> = (0..num_accounts) - .map(|_| { - let account_id = generate_account_id(&mut seed); - let commitment = generate_word(&mut seed); - if account_ids.len() < num_accounts { - account_ids.push(account_id); - } - (account_id, commitment) - }) - .collect(); - - account_tree_hist.compute_and_apply_mutations(mutations).unwrap(); - } - - (account_tree_hist, account_ids) -} - -// VANILLA ACCOUNTTREE BENCHMARKS -// ================================================================================================ - -/// Benchmarks vanilla `AccountTree` open (query) operations. -/// This provides a baseline for comparison with historical access operations. -fn bench_vanilla_access(c: &mut Criterion) { - let mut group = c.benchmark_group("account_tree_vanilla_access"); - - let account_counts = [1, 10, 50, 100, 500, 1000]; - - for &num_accounts in &account_counts { - let (tree, account_ids) = setup_vanilla_account_tree(num_accounts); - - group.bench_function(BenchmarkId::new("vanilla", num_accounts), |b| { - let test_account = *account_ids.first().unwrap(); - b.iter(|| { - tree.open(black_box(test_account)); - }); - }); - } - - group.finish(); -} - -/// Benchmarks vanilla `AccountTree` insertion (mutation) performance. -/// This provides a baseline for comparison with history-tracking insertion. -fn bench_vanilla_insertion(c: &mut Criterion) { - let mut group = c.benchmark_group("account_tree_insertion"); - - let account_counts = [1, 10, 50, 100, 500]; - - for &num_accounts in &account_counts { - group.bench_function(BenchmarkId::new("vanilla", num_accounts), |b| { - b.iter(|| { - let mut seed = [0u8; 32]; - let storage = setup_storage(); - let smt = LargeSmt::with_entries(storage, std::iter::empty()) - .expect("Failed to create empty LargeSmt"); - let mut tree = AccountTree::new(smt).expect("Failed to create AccountTree"); - let entries: Vec<_> = (0..num_accounts) - .map(|_| { - let account_id = generate_account_id(&mut seed); - let commitment = generate_word(&mut seed); - (account_id, commitment) - }) - .collect(); - let mutations = tree.compute_mutations(black_box(entries)).unwrap(); - tree.apply_mutations(black_box(mutations)).unwrap(); - }); - }); - } - - group.finish(); -} - -// HISTORICAL ACCOUNTTREE BENCHMARKS -// ================================================================================================ - -/// Benchmarks historical access at different depths and account counts. -fn bench_historical_access(c: &mut Criterion) { - let mut group = c.benchmark_group("account_tree_historical_access"); - - let account_counts = [10, 100, 500, 2500]; - let block_depths = [0, 5, 10, 20, 32]; - - for &num_accounts in &account_counts { - for &block_depth in &block_depths { - if block_depth > AccountTreeWithHistory::::MAX_HISTORY { - continue; - } - - let (tree_hist, account_ids) = - setup_account_tree_with_history(num_accounts, block_depth + 1); - let current_block = tree_hist.block_number_latest(); - let target_block = current_block - .checked_sub(u32::try_from(block_depth).unwrap()) - .unwrap_or(BlockNumber::GENESIS); - - if block_depth >= tree_hist.history_len() && block_depth > 0 { - continue; - } - - group.bench_function( - BenchmarkId::new(format!("depth_{block_depth}"), num_accounts), - |b| { - let test_account = *account_ids.first().unwrap(); - b.iter(|| { - tree_hist.open_at(black_box(test_account), black_box(target_block)); - }); - }, - ); - } - } - - group.finish(); -} - -/// Benchmarks insertion performance with history tracking at different account counts. -fn bench_insertion_with_history(c: &mut Criterion) { - let mut group = c.benchmark_group("account_tree_insertion"); - - let account_counts = [1, 10, 50, 100, 500, 2500]; - - for &num_accounts in &account_counts { - group.bench_function(BenchmarkId::new("with_history", num_accounts), |b| { - b.iter(|| { - let mut seed = [0u8; 32]; - let storage = setup_storage(); - let smt = LargeSmt::with_entries(storage, std::iter::empty()) - .expect("Failed to create empty LargeSmt"); - let account_tree = AccountTree::new(smt).expect("Failed to create AccountTree"); - let mut tree = AccountTreeWithHistory::new(account_tree, BlockNumber::GENESIS); - let mutations: Vec<_> = (0..num_accounts) - .map(|_| { - let account_id = generate_account_id(&mut seed); - let commitment = generate_word(&mut seed); - (account_id, commitment) - }) - .collect(); - tree.compute_and_apply_mutations(black_box(mutations)).unwrap(); - }); - }); - } - - group.finish(); -} - -criterion_group!( - name = historical_account_tree; - config = Criterion::default() - .measurement_time(std::time::Duration::from_millis(1500)) - .warm_up_time(std::time::Duration::from_millis(100)) - .sample_size(10); - targets = bench_vanilla_access, - bench_vanilla_insertion, - bench_historical_access, - bench_insertion_with_history -); -criterion_main!(historical_account_tree); diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index c0a37be32..7fa5be8e7 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -29,6 +29,10 @@ mod tests; /// Convenience for an in-memory-only account tree. pub type InMemoryAccountTree = AccountTree>; +#[cfg(feature = "rocksdb")] +/// Convenience for a persistent account tree. +pub type PersistentAccountTree = AccountTree>; + // HISTORICAL ERROR TYPES // ================================================================================================ diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 5a9dc5ee2..1d345dcf0 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -7,6 +7,8 @@ mod inner_forest; mod server; pub mod state; +#[cfg(feature = "rocksdb")] +pub use accounts::PersistentAccountTree; pub use accounts::{AccountTreeWithHistory, HistoricalError, InMemoryAccountTree}; pub use genesis::GenesisState; pub use server::{DataDirectory, Store}; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index d9594a87c..875bd23ff 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -8,6 +8,8 @@ use std::ops::RangeInclusive; use std::path::Path; use std::sync::Arc; +#[cfg(feature = "rocksdb")] +use miden_crypto::merkle::smt::RocksDbStorage; use miden_node_proto::domain::account::{ AccountDetailRequest, AccountDetails, @@ -31,6 +33,8 @@ use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_i use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; +#[cfg(feature = "rocksdb")] +use miden_protocol::crypto::merkle::smt::RocksDbConfig; use miden_protocol::crypto::merkle::smt::{ LargeSmt, LargeSmtError, @@ -82,6 +86,33 @@ pub struct TransactionInputs { pub new_account_id_prefix_is_unique: Option, } +/// The storage backend for account trees. +#[cfg(feature = "rocksdb")] +pub type AccountTreeStorage = RocksDbStorage; +#[cfg(not(feature = "rocksdb"))] +pub type AccountTreeStorage = MemoryStorage; + +/// Helper for constructing storage backends. +struct StorageBuilder(std::marker::PhantomData); + +impl StorageBuilder { + #[allow(dead_code, clippy::unnecessary_wraps)] + fn create(_data_dir: &Path) -> Result { + Ok(MemoryStorage::default()) + } +} + +#[cfg(feature = "rocksdb")] +impl StorageBuilder { + fn create(data_dir: &Path) -> Result { + let storage_path = data_dir.join("account_tree"); + fs_err::create_dir_all(&storage_path) + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; + RocksDbStorage::open(RocksDbConfig::new(storage_path)) + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string())) + } +} + /// Container for state that needs to be updated atomically. struct InnerState where @@ -92,10 +123,7 @@ where account_tree: AccountTreeWithHistory, } -impl InnerState -where - S: SmtStorage, -{ +impl InnerState { /// Returns the latest block number. fn latest_block_num(&self) -> BlockNumber { self.blockchain @@ -119,7 +147,7 @@ pub struct State { /// Read-write lock used to prevent writing to a structure while it is being used. /// /// The lock is writer-preferring, meaning the writer won't be starved. - inner: RwLock, + inner: RwLock>, /// Forest-related state `(SmtForest, storage_map_roots, vault_roots)` with its own lock. forest: RwLock, @@ -151,7 +179,10 @@ impl State { let blockchain = load_mmr(&mut db).await?; let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); - let account_tree = load_account_tree(&mut db, latest_block_num).await?; + + let storage = StorageBuilder::::create(data_path)?; + let account_tree = load_account_tree(&mut db, latest_block_num, storage).await?; + let nullifier_tree = load_nullifier_tree(&mut db).await?; let forest = load_smt_forest(&mut db, latest_block_num).await?; @@ -1223,10 +1254,11 @@ async fn load_nullifier_tree( } #[instrument(level = "info", target = COMPONENT, skip_all)] -async fn load_account_tree( +async fn load_account_tree( db: &mut Db, block_number: BlockNumber, -) -> Result, StateInitializationError> { + storage: S, +) -> Result, StateInitializationError> { let account_data = Vec::from_iter(db.select_all_account_commitments().await?); let smt_entries = account_data From 388cf7084643d345922fb6b9a22718b7767c1353 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 17:22:32 +0100 Subject: [PATCH 02/23] CI --- .github/actions/install-rocksdb/action.yml | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/actions/install-rocksdb/action.yml diff --git a/.github/actions/install-rocksdb/action.yml b/.github/actions/install-rocksdb/action.yml new file mode 100644 index 000000000..276d3f9bb --- /dev/null +++ b/.github/actions/install-rocksdb/action.yml @@ -0,0 +1,24 @@ +name: "Install RocksDB" +description: "Install RocksDB dependencies - prefers system library over compiling from source" + +runs: + using: "composite" + steps: + - name: Install RocksDB system library and LLVM/Clang + shell: bash + run: | + set -eux + sudo apt-get update + # Install system rocksdb library to avoid compiling from source. + # Also install clang/llvm for bindgen (still needed for FFI bindings). + sudo apt-get install -y librocksdb-dev clang llvm-dev libclang-dev + + # Point librocksdb-sys to use system library instead of compiling. + # This saves ~5-10 minutes of compile time and reduces cache size. + echo "ROCKSDB_LIB_DIR=/usr/lib" >> "$GITHUB_ENV" + echo "ROCKSDB_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV" + # Use static linking to avoid runtime library issues + echo "ROCKSDB_STATIC=1" >> "$GITHUB_ENV" + + # Required for bindgen + echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> "$GITHUB_ENV" From d99fef7c83fda3158ba65425ecc9e64937f4097f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 17:44:32 +0100 Subject: [PATCH 03/23] Apply suggestion from @Mirko-von-Leipzig Co-authored-by: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> --- crates/store/src/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 875bd23ff..d81cbabdc 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -96,7 +96,7 @@ pub type AccountTreeStorage = MemoryStorage; struct StorageBuilder(std::marker::PhantomData); impl StorageBuilder { - #[allow(dead_code, clippy::unnecessary_wraps)] + #[allow(dead_code, clippy::unnecessary_wraps, reason = "Result is required by its `RocksDb` counterpart")] fn create(_data_dir: &Path) -> Result { Ok(MemoryStorage::default()) } From e4d11eba57c245e67665cd78bc7f630380c37327 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 17:51:08 +0100 Subject: [PATCH 04/23] disable account reconstruction more systematically --- Cargo.toml | 3 +- bin/node/Cargo.toml | 2 +- bin/stress-test/Cargo.toml | 2 +- crates/block-producer/Cargo.toml | 2 +- crates/rpc/Cargo.toml | 2 +- crates/store/src/state.rs | 72 ++++++++++++++++++++++++-------- 6 files changed, 60 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c1cf8f3da..d87c0abf6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ miden-node-ntx-builder = { path = "crates/ntx-builder", version = "0.13" } miden-node-proto = { path = "crates/proto", version = "0.13" } miden-node-proto-build = { path = "proto", version = "0.13" } miden-node-rpc = { path = "crates/rpc", version = "0.13" } -miden-node-store = { path = "crates/store", version = "0.13" } +miden-node-store = { default-features = false, path = "crates/store", version = "0.13" } miden-node-test-macro = { path = "crates/test-macro" } miden-node-utils = { path = "crates/utils", version = "0.13" } miden-node-validator = { path = "crates/validator", version = "0.13" } @@ -58,6 +58,7 @@ miden-tx-batch-prover = { branch = "next", git = "https://github.com/0xMiden/mid # Other miden dependencies. These should align with those expected by miden-base. miden-air = { features = ["std", "testing"], version = "0.20" } +miden-crypto = { default-features = false, version = "0.19" } # External dependencies anyhow = { version = "1.0" } diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index c7a126c97..87ad6e286 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -26,7 +26,7 @@ humantime = { workspace = true } miden-node-block-producer = { workspace = true } miden-node-ntx-builder = { workspace = true } miden-node-rpc = { workspace = true } -miden-node-store = { workspace = true } +miden-node-store = { features = ["rocksdb"], workspace = true } miden-node-utils = { workspace = true } miden-node-validator = { workspace = true } miden-protocol = { workspace = true } diff --git a/bin/stress-test/Cargo.toml b/bin/stress-test/Cargo.toml index b9df84d41..1e96d2ba3 100644 --- a/bin/stress-test/Cargo.toml +++ b/bin/stress-test/Cargo.toml @@ -24,7 +24,7 @@ miden-air = { features = ["testing"], workspace = true } miden-block-prover = { features = ["testing"], workspace = true } miden-node-block-producer = { workspace = true } miden-node-proto = { workspace = true } -miden-node-store = { workspace = true } +miden-node-store = { features = ["rocksdb"], workspace = true } miden-node-utils = { workspace = true } miden-protocol = { workspace = true } miden-standards = { workspace = true } diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index e5e5511ad..a8f5493d4 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -43,7 +43,7 @@ url = { workspace = true } [dev-dependencies] assert_matches = { workspace = true } -miden-node-store = { workspace = true } +miden-node-store = { default-features = false, workspace = true } miden-node-test-macro = { workspace = true } miden-node-utils = { features = ["testing"], workspace = true } miden-protocol = { default-features = true, features = ["testing"], workspace = true } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 30ec4dcb8..133875b22 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -38,7 +38,7 @@ url = { workspace = true } [dev-dependencies] miden-air = { features = ["testing"], workspace = true } -miden-node-store = { workspace = true } +miden-node-store = { default-features = false, workspace = true } miden-node-utils = { features = ["testing", "tracing-forest"], workspace = true } miden-protocol = { default-features = true, features = ["testing"], workspace = true } miden-standards = { workspace = true } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index d81cbabdc..17d6e778f 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -95,6 +95,58 @@ pub type AccountTreeStorage = MemoryStorage; /// Helper for constructing storage backends. struct StorageBuilder(std::marker::PhantomData); +/// Trait for loading account trees from storage. +/// +/// For `MemoryStorage`, the tree is rebuilt from database entries on each startup. +/// For `RocksDbStorage`, the tree is loaded directly from disk (much faster for large trees). +trait AccountTreeLoader: SmtStorage + Sized { + /// Loads an account tree, either from persistent storage or by rebuilding from DB. + fn load_account_tree( + self, + db: &mut Db, + ) -> impl std::future::Future, StateInitializationError>> + Send; +} + +impl AccountTreeLoader for MemoryStorage { + async fn load_account_tree( + self, + db: &mut Db, + ) -> Result, StateInitializationError> { + let account_data = db.select_all_account_commitments().await?; + let smt_entries = account_data + .into_iter() + .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); + LargeSmt::with_entries(self, smt_entries).map_err(|e| match e { + LargeSmtError::Merkle(merkle_error) => { + StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) + }, + LargeSmtError::Storage(err) => { + StateInitializationError::AccountTreeIoError(err.as_report()) + }, + }) + } +} + +#[cfg(feature = "rocksdb")] +impl AccountTreeLoader for RocksDbStorage { + async fn load_account_tree( + self, + _db: &mut Db, + ) -> Result, StateInitializationError> { + // Load directly from RocksDB storage - no need to query DB. + // LargeSmt::new() checks if storage has data and reconstructs from it. + // The tree state is persisted to disk and survives restarts. + LargeSmt::new(self).map_err(|e| match e { + LargeSmtError::Merkle(merkle_error) => { + StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) + }, + LargeSmtError::Storage(err) => { + StateInitializationError::AccountTreeIoError(err.as_report()) + }, + }) + } +} + impl StorageBuilder { #[allow(dead_code, clippy::unnecessary_wraps, reason = "Result is required by its `RocksDb` counterpart")] fn create(_data_dir: &Path) -> Result { @@ -1254,28 +1306,12 @@ async fn load_nullifier_tree( } #[instrument(level = "info", target = COMPONENT, skip_all)] -async fn load_account_tree( +async fn load_account_tree( db: &mut Db, block_number: BlockNumber, storage: S, ) -> Result, StateInitializationError> { - let account_data = Vec::from_iter(db.select_all_account_commitments().await?); - - let smt_entries = account_data - .into_iter() - .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); - - let smt = - LargeSmt::with_entries(MemoryStorage::default(), smt_entries).map_err(|e| match e { - LargeSmtError::Merkle(merkle_error) => { - StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) - }, - LargeSmtError::Storage(err) => { - // large_smt::StorageError is not `Sync` and hence `context` cannot be called - // which we want to and do - StateInitializationError::AccountTreeIoError(err.as_report()) - }, - })?; + let smt = storage.load_account_tree(db).await?; let account_tree = AccountTree::new(smt).map_err(StateInitializationError::FailedToCreateAccountsTree)?; From 132915b4c628618c88bf59e8c7c7093bcfb53f4b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 17:51:38 +0100 Subject: [PATCH 05/23] lost and found --- crates/store/benches/account_tree.rs | 266 +++++++++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 crates/store/benches/account_tree.rs diff --git a/crates/store/benches/account_tree.rs b/crates/store/benches/account_tree.rs new file mode 100644 index 000000000..8c3f1009e --- /dev/null +++ b/crates/store/benches/account_tree.rs @@ -0,0 +1,266 @@ +use std::hint::black_box; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use miden_crypto::merkle::smt::{RocksDbConfig, RocksDbStorage}; +use miden_node_store::AccountTreeWithHistory; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::BlockNumber; +use miden_protocol::block::account_tree::{AccountTree, account_id_to_smt_key}; +use miden_protocol::crypto::hash::rpo::Rpo256; +use miden_protocol::crypto::merkle::smt::LargeSmt; +use miden_protocol::testing::account_id::AccountIdBuilder; + +/// Counter for creating unique `RocksDB` directories during benchmarking. +static DB_COUNTER: AtomicUsize = AtomicUsize::new(0); + +// HELPER FUNCTIONS +// ================================================================================================ + +/// Returns the default base path for `RocksDB` benchmark storage. +fn default_storage_path() -> std::path::PathBuf { + std::path::PathBuf::from("target/bench_rocksdb") +} + +/// Creates a `RocksDB` storage instance for benchmarking. +/// +/// # Arguments +/// * `base_path` - Base directory for `RocksDB` storage. Each call creates a unique subdirectory. +fn setup_storage(base_path: &Path) -> RocksDbStorage { + let counter = DB_COUNTER.fetch_add(1, Ordering::SeqCst); + let db_path = base_path.join(format!("bench_rocksdb_{counter}")); + + // Clean up the directory if it exists + if db_path.exists() { + fs_err::remove_dir_all(&db_path).ok(); + } + fs_err::create_dir_all(&db_path).expect("Failed to create storage directory"); + + RocksDbStorage::open(RocksDbConfig::new(db_path)).expect("RocksDB failed to open file") +} + +/// Generates a deterministic word from a seed. +fn generate_word(seed: &mut [u8; 32]) -> Word { + for byte in seed.iter_mut() { + *byte = byte.wrapping_add(1); + if *byte != 0 { + break; + } + } + Rpo256::hash(seed) +} + +/// Generates a deterministic `AccountId` from a seed. +fn generate_account_id(seed: &mut [u8; 32]) -> AccountId { + for byte in seed.iter_mut() { + *byte = byte.wrapping_add(1); + if *byte != 0 { + break; + } + } + AccountIdBuilder::new().build_with_seed(*seed) +} + +// SETUP FUNCTIONS +// ================================================================================================ + +/// Sets up a vanilla `AccountTree` with specified number of accounts. +fn setup_vanilla_account_tree( + num_accounts: usize, + base_path: &Path, +) -> (AccountTree>, Vec) { + let mut seed = [0u8; 32]; + let mut account_ids = Vec::new(); + let mut entries = Vec::new(); + + for _ in 0..num_accounts { + let account_id = generate_account_id(&mut seed); + let commitment = generate_word(&mut seed); + account_ids.push(account_id); + entries.push((account_id_to_smt_key(account_id), commitment)); + } + + let storage = setup_storage(base_path); + let smt = + LargeSmt::with_entries(storage, entries).expect("Failed to create LargeSmt from entries"); + let tree = AccountTree::new(smt).expect("Failed to create AccountTree"); + (tree, account_ids) +} + +/// Sets up `AccountTreeWithHistory` with specified number of accounts and blocks. +fn setup_account_tree_with_history( + num_accounts: usize, + num_blocks: usize, + base_path: &Path, +) -> (AccountTreeWithHistory, Vec) { + let mut seed = [0u8; 32]; + let storage = setup_storage(base_path); + let smt = LargeSmt::with_entries(storage, std::iter::empty()) + .expect("Failed to create empty LargeSmt"); + let account_tree = AccountTree::new(smt).expect("Failed to create AccountTree"); + let mut account_tree_hist = AccountTreeWithHistory::new(account_tree, BlockNumber::GENESIS); + let mut account_ids = Vec::new(); + + for _block in 0..num_blocks { + let mutations: Vec<_> = (0..num_accounts) + .map(|_| { + let account_id = generate_account_id(&mut seed); + let commitment = generate_word(&mut seed); + if account_ids.len() < num_accounts { + account_ids.push(account_id); + } + (account_id, commitment) + }) + .collect(); + + account_tree_hist.compute_and_apply_mutations(mutations).unwrap(); + } + + (account_tree_hist, account_ids) +} + +// VANILLA ACCOUNTTREE BENCHMARKS +// ================================================================================================ + +/// Benchmarks vanilla `AccountTree` open (query) operations. +/// This provides a baseline for comparison with historical access operations. +fn bench_vanilla_access(c: &mut Criterion) { + let mut group = c.benchmark_group("account_tree_vanilla_access"); + let base_path = default_storage_path(); + + let account_counts = [1, 10, 50, 100, 500, 1000]; + + for &num_accounts in &account_counts { + let (tree, account_ids) = setup_vanilla_account_tree(num_accounts, &base_path); + + group.bench_function(BenchmarkId::new("vanilla", num_accounts), |b| { + let test_account = *account_ids.first().unwrap(); + b.iter(|| { + tree.open(black_box(test_account)); + }); + }); + } + + group.finish(); +} + +/// Benchmarks vanilla `AccountTree` insertion (mutation) performance. +/// This provides a baseline for comparison with history-tracking insertion. +fn bench_vanilla_insertion(c: &mut Criterion) { + let mut group = c.benchmark_group("account_tree_insertion"); + let base_path = default_storage_path(); + + let account_counts = [1, 10, 50, 100, 500]; + + for &num_accounts in &account_counts { + group.bench_function(BenchmarkId::new("vanilla", num_accounts), |b| { + b.iter(|| { + let mut seed = [0u8; 32]; + let storage = setup_storage(&base_path); + let smt = LargeSmt::with_entries(storage, std::iter::empty()) + .expect("Failed to create empty LargeSmt"); + let mut tree = AccountTree::new(smt).expect("Failed to create AccountTree"); + let entries: Vec<_> = (0..num_accounts) + .map(|_| { + let account_id = generate_account_id(&mut seed); + let commitment = generate_word(&mut seed); + (account_id, commitment) + }) + .collect(); + let mutations = tree.compute_mutations(black_box(entries)).unwrap(); + tree.apply_mutations(black_box(mutations)).unwrap(); + }); + }); + } + + group.finish(); +} + +// HISTORICAL ACCOUNTTREE BENCHMARKS +// ================================================================================================ + +/// Benchmarks historical access at different depths and account counts. +fn bench_historical_access(c: &mut Criterion) { + let mut group = c.benchmark_group("account_tree_historical_access"); + let base_path = default_storage_path(); + + let account_counts = [10, 100, 500, 2500]; + let block_depths = [0, 5, 10, 20, 32]; + + for &num_accounts in &account_counts { + for &block_depth in &block_depths { + if block_depth > AccountTreeWithHistory::::MAX_HISTORY { + continue; + } + + let (tree_hist, account_ids) = + setup_account_tree_with_history(num_accounts, block_depth + 1, &base_path); + let current_block = tree_hist.block_number_latest(); + let target_block = current_block + .checked_sub(u32::try_from(block_depth).unwrap()) + .unwrap_or(BlockNumber::GENESIS); + + if block_depth >= tree_hist.history_len() && block_depth > 0 { + continue; + } + + group.bench_function( + BenchmarkId::new(format!("depth_{block_depth}"), num_accounts), + |b| { + let test_account = *account_ids.first().unwrap(); + b.iter(|| { + tree_hist.open_at(black_box(test_account), black_box(target_block)); + }); + }, + ); + } + } + + group.finish(); +} + +/// Benchmarks insertion performance with history tracking at different account counts. +fn bench_insertion_with_history(c: &mut Criterion) { + let mut group = c.benchmark_group("account_tree_insertion"); + let base_path = default_storage_path(); + + let account_counts = [1, 10, 50, 100, 500, 2500]; + + for &num_accounts in &account_counts { + group.bench_function(BenchmarkId::new("with_history", num_accounts), |b| { + b.iter(|| { + let mut seed = [0u8; 32]; + let storage = setup_storage(&base_path); + let smt = LargeSmt::with_entries(storage, std::iter::empty()) + .expect("Failed to create empty LargeSmt"); + let account_tree = AccountTree::new(smt).expect("Failed to create AccountTree"); + let mut tree = AccountTreeWithHistory::new(account_tree, BlockNumber::GENESIS); + let mutations: Vec<_> = (0..num_accounts) + .map(|_| { + let account_id = generate_account_id(&mut seed); + let commitment = generate_word(&mut seed); + (account_id, commitment) + }) + .collect(); + tree.compute_and_apply_mutations(black_box(mutations)).unwrap(); + }); + }); + } + + group.finish(); +} + +criterion_group!( + name = historical_account_tree; + config = Criterion::default() + .measurement_time(std::time::Duration::from_millis(1500)) + .warm_up_time(std::time::Duration::from_millis(100)) + .sample_size(10); + targets = bench_vanilla_access, + bench_vanilla_insertion, + bench_historical_access, + bench_insertion_with_history +); +criterion_main!(historical_account_tree); From 087cb7b504f87902610cc18c0297925acaef45e5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 17:59:37 +0100 Subject: [PATCH 06/23] yay --- crates/store/src/state.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 17d6e778f..ed2bd7c13 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -98,7 +98,9 @@ struct StorageBuilder(std::marker::PhantomData); /// Trait for loading account trees from storage. /// /// For `MemoryStorage`, the tree is rebuilt from database entries on each startup. -/// For `RocksDbStorage`, the tree is loaded directly from disk (much faster for large trees). +/// For `RocksDbStorage`, the tree is _not_ loaded into memory, but we assume the existing file +/// on disk is coherent. +// TODO handle on disk rocksdb storage file being missing and/or corrupted. trait AccountTreeLoader: SmtStorage + Sized { /// Loads an account tree, either from persistent storage or by rebuilding from DB. fn load_account_tree( @@ -107,6 +109,8 @@ trait AccountTreeLoader: SmtStorage + Sized { ) -> impl std::future::Future, StateInitializationError>> + Send; } +// Should only every be used in `cfg(test)` scope! +#[cfg(not(feature = "rocksdb"))] impl AccountTreeLoader for MemoryStorage { async fn load_account_tree( self, @@ -147,8 +151,9 @@ impl AccountTreeLoader for RocksDbStorage { } } +#[cfg(test)] impl StorageBuilder { - #[allow(dead_code, clippy::unnecessary_wraps, reason = "Result is required by its `RocksDb` counterpart")] + #[allow(clippy::unnecessary_wraps)] fn create(_data_dir: &Path) -> Result { Ok(MemoryStorage::default()) } @@ -213,7 +218,7 @@ impl State { // CONSTRUCTOR // -------------------------------------------------------------------------------------------- - /// Loads the state from the `db`. + /// Loads the state from `sqlite` into `MemoryBackend`. #[instrument(target = COMPONENT, skip_all)] pub async fn load(data_path: &Path) -> Result { let data_directory = DataDirectory::load(data_path.to_path_buf()) From 089de41a5819f4d2823089683126f20aaf951ea4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 18:08:17 +0100 Subject: [PATCH 07/23] sanity, please --- crates/store/src/state.rs | 50 ++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index ed2bd7c13..94b995826 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -107,10 +107,16 @@ trait AccountTreeLoader: SmtStorage + Sized { self, db: &mut Db, ) -> impl std::future::Future, StateInitializationError>> + Send; + + /// Loads a nullifier tree, either from persistent storage or by rebuilding from DB. + fn load_nullifier_tree( + self, + db: &mut Db, + ) -> impl std::future::Future>, StateInitializationError>> + + Send; } // Should only every be used in `cfg(test)` scope! -#[cfg(not(feature = "rocksdb"))] impl AccountTreeLoader for MemoryStorage { async fn load_account_tree( self, @@ -129,6 +135,16 @@ impl AccountTreeLoader for MemoryStorage { }, }) } + + async fn load_nullifier_tree( + self, + db: &mut Db, + ) -> Result>, StateInitializationError> { + let nullifiers = db.select_all_nullifiers().await?; + let entries = nullifiers.into_iter().map(|info| (info.nullifier, info.block_num)); + NullifierTree::with_storage_from_entries(self, entries) + .map_err(StateInitializationError::FailedToCreateNullifierTree) + } } #[cfg(feature = "rocksdb")] @@ -149,6 +165,22 @@ impl AccountTreeLoader for RocksDbStorage { }, }) } + + async fn load_nullifier_tree( + self, + _db: &mut Db, + ) -> Result>, StateInitializationError> { + // Load directly from RocksDB storage - no need to query DB. + let smt = LargeSmt::new(self).map_err(|e| match e { + LargeSmtError::Merkle(merkle_error) => { + StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) + }, + LargeSmtError::Storage(err) => { + StateInitializationError::AccountTreeIoError(err.as_report()) + }, + })?; + Ok(NullifierTree::new_unchecked(smt)) + } } #[cfg(test)] @@ -171,7 +203,7 @@ impl StorageBuilder { } /// Container for state that needs to be updated atomically. -struct InnerState +struct InnerState where S: SmtStorage, { @@ -1296,20 +1328,6 @@ async fn load_mmr(db: &mut Db) -> Result { Ok(chain_mmr) } -#[instrument(level = "info", target = COMPONENT, skip_all)] -async fn load_nullifier_tree( - db: &mut Db, -) -> Result>, StateInitializationError> { - let nullifiers = db.select_all_nullifiers().await?; - - // Convert nullifier data to entries for NullifierTree - // The nullifier value format is: block_num - let entries = nullifiers.into_iter().map(|info| (info.nullifier, info.block_num)); - - NullifierTree::with_storage_from_entries(MemoryStorage::default(), entries) - .map_err(StateInitializationError::FailedToCreateNullifierTree) -} - #[instrument(level = "info", target = COMPONENT, skip_all)] async fn load_account_tree( db: &mut Db, From 9faa6633f68e753920a0499f4ad410d421d675e2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 19:31:17 +0100 Subject: [PATCH 08/23] cleanup --- crates/store/src/state.rs | 138 ++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 81 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 94b995826..c6e274ead 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -27,21 +27,17 @@ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; use miden_protocol::Word; -use miden_protocol::account::AccountId; use miden_protocol::account::delta::AccountUpdateDetails; +use miden_protocol::account::{AccountId, StorageSlotContent}; use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; #[cfg(feature = "rocksdb")] use miden_protocol::crypto::merkle::smt::RocksDbConfig; -use miden_protocol::crypto::merkle::smt::{ - LargeSmt, - LargeSmtError, - MemoryStorage, - SmtProof, - SmtStorage, -}; +use miden_protocol::crypto::merkle::smt::{LargeSmt, LargeSmtError, SmtProof, SmtStorage}; +#[cfg(not(feature = "rocksdb"))] +use miden_protocol::crypto::merkle::smt::MemoryStorage; use miden_protocol::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::{OutputNote, PartialBlockchain}; use miden_protocol::utils::Serializable; @@ -86,22 +82,39 @@ pub struct TransactionInputs { pub new_account_id_prefix_is_unique: Option, } -/// The storage backend for account trees. +/// The storage backend for trees. #[cfg(feature = "rocksdb")] -pub type AccountTreeStorage = RocksDbStorage; +pub type TreeStorage = RocksDbStorage; #[cfg(not(feature = "rocksdb"))] -pub type AccountTreeStorage = MemoryStorage; +pub type TreeStorage = MemoryStorage; + +/// Converts a `LargeSmtError` into a `StateInitializationError`. +fn large_smt_error_to_init_error(e: LargeSmtError) -> StateInitializationError { + match e { + LargeSmtError::Merkle(merkle_error) => { + StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) + }, + LargeSmtError::Storage(err) => { + StateInitializationError::AccountTreeIoError(err.as_report()) + }, + } +} -/// Helper for constructing storage backends. -struct StorageBuilder(std::marker::PhantomData); +/// Loads an SMT from persistent storage. +#[cfg(feature = "rocksdb")] +fn load_smt(storage: S) -> Result, StateInitializationError> { + LargeSmt::new(storage).map_err(large_smt_error_to_init_error) +} -/// Trait for loading account trees from storage. +/// Trait for loading trees from storage. /// /// For `MemoryStorage`, the tree is rebuilt from database entries on each startup. -/// For `RocksDbStorage`, the tree is _not_ loaded into memory, but we assume the existing file -/// on disk is coherent. +/// For `RocksDbStorage`, the tree is loaded directly from disk (much faster for large trees). // TODO handle on disk rocksdb storage file being missing and/or corrupted. -trait AccountTreeLoader: SmtStorage + Sized { +trait StorageLoader: SmtStorage + Sized { + /// Creates a storage backend for the given domain. + fn create(data_dir: &Path, domain: &'static str) -> Result; + /// Loads an account tree, either from persistent storage or by rebuilding from DB. fn load_account_tree( self, @@ -116,8 +129,12 @@ trait AccountTreeLoader: SmtStorage + Sized { + Send; } -// Should only every be used in `cfg(test)` scope! -impl AccountTreeLoader for MemoryStorage { +#[cfg(not(feature = "rocksdb"))] +impl StorageLoader for MemoryStorage { + fn create(_data_dir: &Path, _domain: &'static str) -> Result { + Ok(MemoryStorage::default()) + } + async fn load_account_tree( self, db: &mut Db, @@ -126,14 +143,7 @@ impl AccountTreeLoader for MemoryStorage { let smt_entries = account_data .into_iter() .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); - LargeSmt::with_entries(self, smt_entries).map_err(|e| match e { - LargeSmtError::Merkle(merkle_error) => { - StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) - }, - LargeSmtError::Storage(err) => { - StateInitializationError::AccountTreeIoError(err.as_report()) - }, - }) + LargeSmt::with_entries(self, smt_entries).map_err(large_smt_error_to_init_error) } async fn load_nullifier_tree( @@ -148,7 +158,15 @@ impl AccountTreeLoader for MemoryStorage { } #[cfg(feature = "rocksdb")] -impl AccountTreeLoader for RocksDbStorage { +impl StorageLoader for RocksDbStorage { + fn create(data_dir: &Path, domain: &'static str) -> Result { + let storage_path = data_dir.join(domain); + fs_err::create_dir_all(&storage_path) + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; + RocksDbStorage::open(RocksDbConfig::new(storage_path)) + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string())) + } + async fn load_account_tree( self, _db: &mut Db, @@ -156,14 +174,7 @@ impl AccountTreeLoader for RocksDbStorage { // Load directly from RocksDB storage - no need to query DB. // LargeSmt::new() checks if storage has data and reconstructs from it. // The tree state is persisted to disk and survives restarts. - LargeSmt::new(self).map_err(|e| match e { - LargeSmtError::Merkle(merkle_error) => { - StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) - }, - LargeSmtError::Storage(err) => { - StateInitializationError::AccountTreeIoError(err.as_report()) - }, - }) + load_smt(self) } async fn load_nullifier_tree( @@ -171,37 +182,11 @@ impl AccountTreeLoader for RocksDbStorage { _db: &mut Db, ) -> Result>, StateInitializationError> { // Load directly from RocksDB storage - no need to query DB. - let smt = LargeSmt::new(self).map_err(|e| match e { - LargeSmtError::Merkle(merkle_error) => { - StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) - }, - LargeSmtError::Storage(err) => { - StateInitializationError::AccountTreeIoError(err.as_report()) - }, - })?; + let smt = load_smt(self)?; Ok(NullifierTree::new_unchecked(smt)) } } -#[cfg(test)] -impl StorageBuilder { - #[allow(clippy::unnecessary_wraps)] - fn create(_data_dir: &Path) -> Result { - Ok(MemoryStorage::default()) - } -} - -#[cfg(feature = "rocksdb")] -impl StorageBuilder { - fn create(data_dir: &Path) -> Result { - let storage_path = data_dir.join("account_tree"); - fs_err::create_dir_all(&storage_path) - .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; - RocksDbStorage::open(RocksDbConfig::new(storage_path)) - .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string())) - } -} - /// Container for state that needs to be updated atomically. struct InnerState where @@ -236,7 +221,7 @@ pub struct State { /// Read-write lock used to prevent writing to a structure while it is being used. /// /// The lock is writer-preferring, meaning the writer won't be starved. - inner: RwLock>, + inner: RwLock>, /// Forest-related state `(SmtForest, storage_map_roots, vault_roots)` with its own lock. forest: RwLock, @@ -269,10 +254,15 @@ impl State { let blockchain = load_mmr(&mut db).await?; let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); - let storage = StorageBuilder::::create(data_path)?; - let account_tree = load_account_tree(&mut db, latest_block_num, storage).await?; + let account_storage = TreeStorage::create(data_path, "accounttree")?; + let smt = account_storage.load_account_tree(&mut db).await?; + let account_tree = AccountTree::new(smt) + .map_err(StateInitializationError::FailedToCreateAccountsTree)?; + let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); + + let nullifier_storage = TreeStorage::create(data_path, "nullifiertree")?; + let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; - let nullifier_tree = load_nullifier_tree(&mut db).await?; let forest = load_smt_forest(&mut db, latest_block_num).await?; let inner = RwLock::new(InnerState { nullifier_tree, blockchain, account_tree }); @@ -1328,20 +1318,6 @@ async fn load_mmr(db: &mut Db) -> Result { Ok(chain_mmr) } -#[instrument(level = "info", target = COMPONENT, skip_all)] -async fn load_account_tree( - db: &mut Db, - block_number: BlockNumber, - storage: S, -) -> Result, StateInitializationError> { - let smt = storage.load_account_tree(db).await?; - - let account_tree = - AccountTree::new(smt).map_err(StateInitializationError::FailedToCreateAccountsTree)?; - - Ok(AccountTreeWithHistory::new(account_tree, block_number)) -} - /// Loads SMT forest with storage map and vault Merkle paths for all public accounts. #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num))] async fn load_smt_forest( From d8a5dc2d62c85b4fe30fcbfc66e15ac810c65480 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 19:48:33 +0100 Subject: [PATCH 09/23] simplify --- Cargo.lock | 127 ++++++++++++++++++++++++++++++++++++++ crates/store/src/state.rs | 18 +++--- 2 files changed, 138 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ad02438b..aa8a55777 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,6 +342,24 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.114", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -446,6 +464,16 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "camino" version = "1.2.2" @@ -496,6 +524,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cf-rustracing" version = "1.2.1" @@ -605,6 +642,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "3.2.25" @@ -2229,12 +2277,36 @@ version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "librocksdb-sys" +version = "0.17.3+10.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", +] + [[package]] name = "libsqlite3-sys" version = "0.35.0" @@ -2255,6 +2327,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libz-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2367,6 +2450,16 @@ version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2527,6 +2620,7 @@ dependencies = [ "rand_core 0.9.5", "rand_hc", "rayon", + "rocksdb", "sha2", "sha3", "subtle", @@ -2824,6 +2918,7 @@ dependencies = [ "fs-err", "hex", "indexmap 2.13.0", + "miden-crypto", "miden-node-proto", "miden-node-proto-build", "miden-node-test-macro", @@ -3248,6 +3343,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3348,6 +3449,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4626,6 +4737,16 @@ dependencies = [ "serde", ] +[[package]] +name = "rocksdb" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rstest" version = "0.26.1" @@ -4671,6 +4792,12 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.2.3" diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index c6e274ead..bab0960ac 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -29,15 +29,18 @@ use miden_node_utils::formatting::format_array; use miden_protocol::Word; use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::account::{AccountId, StorageSlotContent}; -use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; +#[cfg(not(feature = "rocksdb"))] +use miden_protocol::block::account_tree::account_id_to_smt_key; +use miden_protocol::block::account_tree::{AccountTree, AccountWitness}; +>>>>>>> 45437288 (simplify) use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; +#[cfg(not(feature = "rocksdb"))] +use miden_protocol::crypto::merkle::smt::MemoryStorage; #[cfg(feature = "rocksdb")] use miden_protocol::crypto::merkle::smt::RocksDbConfig; use miden_protocol::crypto::merkle::smt::{LargeSmt, LargeSmtError, SmtProof, SmtStorage}; -#[cfg(not(feature = "rocksdb"))] -use miden_protocol::crypto::merkle::smt::MemoryStorage; use miden_protocol::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::{OutputNote, PartialBlockchain}; use miden_protocol::utils::Serializable; @@ -125,8 +128,9 @@ trait StorageLoader: SmtStorage + Sized { fn load_nullifier_tree( self, db: &mut Db, - ) -> impl std::future::Future>, StateInitializationError>> - + Send; + ) -> impl std::future::Future< + Output = Result>, StateInitializationError>, + > + Send; } #[cfg(not(feature = "rocksdb"))] @@ -256,8 +260,8 @@ impl State { let account_storage = TreeStorage::create(data_path, "accounttree")?; let smt = account_storage.load_account_tree(&mut db).await?; - let account_tree = AccountTree::new(smt) - .map_err(StateInitializationError::FailedToCreateAccountsTree)?; + let account_tree = + AccountTree::new(smt).map_err(StateInitializationError::FailedToCreateAccountsTree)?; let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); let nullifier_storage = TreeStorage::create(data_path, "nullifiertree")?; From 3ca9a5be99ab9b1973740b4326444075b3957285 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 20:05:56 +0100 Subject: [PATCH 10/23] gating --- crates/store/src/db/mod.rs | 2 ++ crates/store/src/db/models/queries/accounts.rs | 1 + crates/store/src/db/models/queries/nullifiers.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index a2dacd235..b74d91062 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -322,6 +322,7 @@ impl Db { } /// Loads all the nullifiers from the DB. + #[cfg(not(feature = "rocksdb"))] #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub(crate) async fn select_all_nullifiers(&self) -> Result> { self.transact("all nullifiers", move |conn| { @@ -393,6 +394,7 @@ impl Db { } /// TODO marked for removal, replace with paged version + #[cfg(not(feature = "rocksdb"))] #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_account_commitments(&self) -> Result> { self.transact("read all account commitments", move |conn| { diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index f517360cd..7e796931d 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -263,6 +263,7 @@ pub(crate) fn select_account_by_id_prefix( /// ORDER BY /// block_num ASC /// ``` +#[cfg(not(feature = "rocksdb"))] pub(crate) fn select_all_account_commitments( conn: &mut SqliteConnection, ) -> Result, DatabaseError> { diff --git a/crates/store/src/db/models/queries/nullifiers.rs b/crates/store/src/db/models/queries/nullifiers.rs index 9b4a1029f..b1557b82b 100644 --- a/crates/store/src/db/models/queries/nullifiers.rs +++ b/crates/store/src/db/models/queries/nullifiers.rs @@ -127,6 +127,7 @@ pub(crate) fn select_nullifiers_by_prefix( /// ORDER BY /// block_num ASC /// ``` +#[cfg(any(not(feature = "rocksdb"), test))] pub(crate) fn select_all_nullifiers( conn: &mut SqliteConnection, ) -> Result, DatabaseError> { From 5362e21ba0ea31c12b48edbdcd08b3859af9f043 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 20:14:31 +0100 Subject: [PATCH 11/23] fixup --- Cargo.toml | 2 +- bin/node/Cargo.toml | 2 +- bin/stress-test/Cargo.toml | 2 +- crates/block-producer/Cargo.toml | 2 +- crates/rpc/Cargo.toml | 2 +- crates/store/src/state.rs | 6 +----- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d87c0abf6..6acc89250 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ miden-node-ntx-builder = { path = "crates/ntx-builder", version = "0.13" } miden-node-proto = { path = "crates/proto", version = "0.13" } miden-node-proto-build = { path = "proto", version = "0.13" } miden-node-rpc = { path = "crates/rpc", version = "0.13" } -miden-node-store = { default-features = false, path = "crates/store", version = "0.13" } +miden-node-store = { path = "crates/store", version = "0.13" } miden-node-test-macro = { path = "crates/test-macro" } miden-node-utils = { path = "crates/utils", version = "0.13" } miden-node-validator = { path = "crates/validator", version = "0.13" } diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index 87ad6e286..c7a126c97 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -26,7 +26,7 @@ humantime = { workspace = true } miden-node-block-producer = { workspace = true } miden-node-ntx-builder = { workspace = true } miden-node-rpc = { workspace = true } -miden-node-store = { features = ["rocksdb"], workspace = true } +miden-node-store = { workspace = true } miden-node-utils = { workspace = true } miden-node-validator = { workspace = true } miden-protocol = { workspace = true } diff --git a/bin/stress-test/Cargo.toml b/bin/stress-test/Cargo.toml index 1e96d2ba3..b9df84d41 100644 --- a/bin/stress-test/Cargo.toml +++ b/bin/stress-test/Cargo.toml @@ -24,7 +24,7 @@ miden-air = { features = ["testing"], workspace = true } miden-block-prover = { features = ["testing"], workspace = true } miden-node-block-producer = { workspace = true } miden-node-proto = { workspace = true } -miden-node-store = { features = ["rocksdb"], workspace = true } +miden-node-store = { workspace = true } miden-node-utils = { workspace = true } miden-protocol = { workspace = true } miden-standards = { workspace = true } diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index a8f5493d4..e5e5511ad 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -43,7 +43,7 @@ url = { workspace = true } [dev-dependencies] assert_matches = { workspace = true } -miden-node-store = { default-features = false, workspace = true } +miden-node-store = { workspace = true } miden-node-test-macro = { workspace = true } miden-node-utils = { features = ["testing"], workspace = true } miden-protocol = { default-features = true, features = ["testing"], workspace = true } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 133875b22..30ec4dcb8 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -38,7 +38,7 @@ url = { workspace = true } [dev-dependencies] miden-air = { features = ["testing"], workspace = true } -miden-node-store = { default-features = false, workspace = true } +miden-node-store = { workspace = true } miden-node-utils = { features = ["testing", "tracing-forest"], workspace = true } miden-protocol = { default-features = true, features = ["testing"], workspace = true } miden-standards = { workspace = true } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index bab0960ac..0c5e505b3 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -175,9 +175,6 @@ impl StorageLoader for RocksDbStorage { self, _db: &mut Db, ) -> Result, StateInitializationError> { - // Load directly from RocksDB storage - no need to query DB. - // LargeSmt::new() checks if storage has data and reconstructs from it. - // The tree state is persisted to disk and survives restarts. load_smt(self) } @@ -185,7 +182,6 @@ impl StorageLoader for RocksDbStorage { self, _db: &mut Db, ) -> Result>, StateInitializationError> { - // Load directly from RocksDB storage - no need to query DB. let smt = load_smt(self)?; Ok(NullifierTree::new_unchecked(smt)) } @@ -239,7 +235,7 @@ impl State { // CONSTRUCTOR // -------------------------------------------------------------------------------------------- - /// Loads the state from `sqlite` into `MemoryBackend`. + /// Loads the state from the data directory. #[instrument(target = COMPONENT, skip_all)] pub async fn load(data_path: &Path) -> Result { let data_directory = DataDirectory::load(data_path.to_path_buf()) From cf6c342b3d3456d7497e6c19513d7156b9473790 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 20:30:55 +0100 Subject: [PATCH 12/23] fixin --- .github/actions/install-rocksdb/action.yml | 1 - .github/workflows/msrv.yml | 39 ++++++++++++++++++++++ .github/workflows/publish-debian-all.yml | 2 ++ .github/workflows/publish-debian.yml | 3 ++ .github/workflows/publish-dry-run.yml | 2 ++ .github/workflows/publish-main.yml | 2 ++ .github/workflows/stress-test-check.yml | 2 ++ .github/workflows/test-beta.yml | 2 ++ bin/node/Dockerfile | 5 ++- 9 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/msrv.yml diff --git a/.github/actions/install-rocksdb/action.yml b/.github/actions/install-rocksdb/action.yml index 276d3f9bb..97c8ac0c4 100644 --- a/.github/actions/install-rocksdb/action.yml +++ b/.github/actions/install-rocksdb/action.yml @@ -14,7 +14,6 @@ runs: sudo apt-get install -y librocksdb-dev clang llvm-dev libclang-dev # Point librocksdb-sys to use system library instead of compiling. - # This saves ~5-10 minutes of compile time and reduces cache size. echo "ROCKSDB_LIB_DIR=/usr/lib" >> "$GITHUB_ENV" echo "ROCKSDB_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV" # Use static linking to avoid runtime library issues diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml new file mode 100644 index 000000000..a5f383dab --- /dev/null +++ b/.github/workflows/msrv.yml @@ -0,0 +1,39 @@ +name: Check MSRV + +on: + push: + branches: [next] + pull_request: + types: [opened, reopened, synchronize] + +# Limits workflow concurrency to only the latest commit in the PR. +concurrency: + group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}" + cancel-in-progress: true + +permissions: + contents: read + +env: + # Reduce cache usage by removing debug information. + CARGO_PROFILE_DEV_DEBUG: 0 + +jobs: + # Check MSRV (aka `rust-version`) in `Cargo.toml` is valid for workspace members + msrv: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y jq + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }} + - name: Install cargo-msrv + run: cargo install cargo-msrv + - name: Check MSRV for each workspace member + run: | + ./scripts/check-msrv.sh diff --git a/.github/workflows/publish-debian-all.yml b/.github/workflows/publish-debian-all.yml index 1539d7b1f..413f60d1d 100644 --- a/.github/workflows/publish-debian-all.yml +++ b/.github/workflows/publish-debian-all.yml @@ -31,6 +31,8 @@ jobs: uses: actions/checkout@main with: fetch-depth: 0 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Build and Publish Node uses: ./.github/actions/debian with: diff --git a/.github/workflows/publish-debian.yml b/.github/workflows/publish-debian.yml index 1079bfddb..1d951fe82 100644 --- a/.github/workflows/publish-debian.yml +++ b/.github/workflows/publish-debian.yml @@ -60,6 +60,9 @@ jobs: with: fetch-depth: 0 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Build and Publish Packages uses: ./.github/actions/debian with: diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml index 2acaab2fa..10841d8d3 100644 --- a/.github/workflows/publish-dry-run.yml +++ b/.github/workflows/publish-dry-run.yml @@ -23,6 +23,8 @@ jobs: fetch-depth: 0 - name: Cleanup large tools for build space uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y jq - name: Update Rust toolchain diff --git a/.github/workflows/publish-main.yml b/.github/workflows/publish-main.yml index 25fe4552c..4e46ca497 100644 --- a/.github/workflows/publish-main.yml +++ b/.github/workflows/publish-main.yml @@ -18,6 +18,8 @@ jobs: with: fetch-depth: 0 ref: main + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb # Ensure the release tag refers to the latest commit on main. # Compare the commit SHA that triggered the workflow with the HEAD of the branch we just # checked out (main). diff --git a/.github/workflows/stress-test-check.yml b/.github/workflows/stress-test-check.yml index 47182f8f9..d543d8c83 100644 --- a/.github/workflows/stress-test-check.yml +++ b/.github/workflows/stress-test-check.yml @@ -27,6 +27,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@main + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/test-beta.yml b/.github/workflows/test-beta.yml index 042d50be2..146e68751 100644 --- a/.github/workflows/test-beta.yml +++ b/.github/workflows/test-beta.yml @@ -16,6 +16,8 @@ jobs: - uses: actions/checkout@v4 with: ref: 'next' + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: rustup install beta && rustup default beta - uses: taiki-e/install-action@nextest diff --git a/bin/node/Dockerfile b/bin/node/Dockerfile index 3becd3ded..069c8e32c 100644 --- a/bin/node/Dockerfile +++ b/bin/node/Dockerfile @@ -2,9 +2,12 @@ FROM rust:1.90-slim-bullseye AS builder RUN apt-get update && \ apt-get -y upgrade && \ - apt-get install -y llvm clang bindgen pkg-config libssl-dev libsqlite3-dev ca-certificates && \ + apt-get install -y llvm clang bindgen pkg-config libssl-dev libsqlite3-dev librocksdb-dev ca-certificates && \ rm -rf /var/lib/apt/lists/* +ENV ROCKSDB_LIB_DIR=/usr/lib +ENV ROCKSDB_INCLUDE_DIR=/usr/include + WORKDIR /app COPY ./Cargo.toml . COPY ./Cargo.lock . From 90ec80bebbfad94cc886d89a955a341ed5dbc673 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 15 Jan 2026 21:43:01 +0100 Subject: [PATCH 13/23] rocksdb system libs too old, debian; populate if rocksdb is empty --- .github/actions/install-rocksdb/action.yml | 19 +++------- bin/node/Dockerfile | 6 +-- crates/store/src/db/mod.rs | 2 - .../store/src/db/models/queries/accounts.rs | 1 - .../store/src/db/models/queries/nullifiers.rs | 1 - crates/store/src/state.rs | 38 ++++++++++++++++--- 6 files changed, 39 insertions(+), 28 deletions(-) diff --git a/.github/actions/install-rocksdb/action.yml b/.github/actions/install-rocksdb/action.yml index 97c8ac0c4..62b32b198 100644 --- a/.github/actions/install-rocksdb/action.yml +++ b/.github/actions/install-rocksdb/action.yml @@ -1,23 +1,14 @@ name: "Install RocksDB" -description: "Install RocksDB dependencies - prefers system library over compiling from source" +description: "Install dependencies for RocksDB compilation" runs: using: "composite" steps: - - name: Install RocksDB system library and LLVM/Clang + - name: Install LLVM/Clang for RocksDB shell: bash run: | set -eux sudo apt-get update - # Install system rocksdb library to avoid compiling from source. - # Also install clang/llvm for bindgen (still needed for FFI bindings). - sudo apt-get install -y librocksdb-dev clang llvm-dev libclang-dev - - # Point librocksdb-sys to use system library instead of compiling. - echo "ROCKSDB_LIB_DIR=/usr/lib" >> "$GITHUB_ENV" - echo "ROCKSDB_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV" - # Use static linking to avoid runtime library issues - echo "ROCKSDB_STATIC=1" >> "$GITHUB_ENV" - - # Required for bindgen - echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> "$GITHUB_ENV" + # Install clang/llvm for bindgen (needed for FFI bindings). + # RocksDB is compiled from source by librocksdb-sys. + sudo apt-get install -y clang llvm-dev libclang-dev diff --git a/bin/node/Dockerfile b/bin/node/Dockerfile index 069c8e32c..832b0bb8d 100644 --- a/bin/node/Dockerfile +++ b/bin/node/Dockerfile @@ -1,13 +1,11 @@ FROM rust:1.90-slim-bullseye AS builder +# Install build dependencies. RocksDB is compiled from source by librocksdb-sys. RUN apt-get update && \ apt-get -y upgrade && \ - apt-get install -y llvm clang bindgen pkg-config libssl-dev libsqlite3-dev librocksdb-dev ca-certificates && \ + apt-get install -y llvm clang libclang-dev pkg-config libssl-dev libsqlite3-dev ca-certificates && \ rm -rf /var/lib/apt/lists/* -ENV ROCKSDB_LIB_DIR=/usr/lib -ENV ROCKSDB_INCLUDE_DIR=/usr/include - WORKDIR /app COPY ./Cargo.toml . COPY ./Cargo.lock . diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index b74d91062..a2dacd235 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -322,7 +322,6 @@ impl Db { } /// Loads all the nullifiers from the DB. - #[cfg(not(feature = "rocksdb"))] #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub(crate) async fn select_all_nullifiers(&self) -> Result> { self.transact("all nullifiers", move |conn| { @@ -394,7 +393,6 @@ impl Db { } /// TODO marked for removal, replace with paged version - #[cfg(not(feature = "rocksdb"))] #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_account_commitments(&self) -> Result> { self.transact("read all account commitments", move |conn| { diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 7e796931d..f517360cd 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -263,7 +263,6 @@ pub(crate) fn select_account_by_id_prefix( /// ORDER BY /// block_num ASC /// ``` -#[cfg(not(feature = "rocksdb"))] pub(crate) fn select_all_account_commitments( conn: &mut SqliteConnection, ) -> Result, DatabaseError> { diff --git a/crates/store/src/db/models/queries/nullifiers.rs b/crates/store/src/db/models/queries/nullifiers.rs index b1557b82b..9b4a1029f 100644 --- a/crates/store/src/db/models/queries/nullifiers.rs +++ b/crates/store/src/db/models/queries/nullifiers.rs @@ -127,7 +127,6 @@ pub(crate) fn select_nullifiers_by_prefix( /// ORDER BY /// block_num ASC /// ``` -#[cfg(any(not(feature = "rocksdb"), test))] pub(crate) fn select_all_nullifiers( conn: &mut SqliteConnection, ) -> Result, DatabaseError> { diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 0c5e505b3..284f9f4bb 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -32,13 +32,14 @@ use miden_protocol::account::{AccountId, StorageSlotContent}; #[cfg(not(feature = "rocksdb"))] use miden_protocol::block::account_tree::account_id_to_smt_key; use miden_protocol::block::account_tree::{AccountTree, AccountWitness}; ->>>>>>> 45437288 (simplify) use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; #[cfg(not(feature = "rocksdb"))] use miden_protocol::crypto::merkle::smt::MemoryStorage; #[cfg(feature = "rocksdb")] +use tracing::warn; +#[cfg(feature = "rocksdb")] use miden_protocol::crypto::merkle::smt::RocksDbConfig; use miden_protocol::crypto::merkle::smt::{LargeSmt, LargeSmtError, SmtProof, SmtStorage}; use miden_protocol::note::{NoteDetails, NoteId, NoteScript, Nullifier}; @@ -173,17 +174,42 @@ impl StorageLoader for RocksDbStorage { async fn load_account_tree( self, - _db: &mut Db, + db: &mut Db, ) -> Result, StateInitializationError> { - load_smt(self) + // If RocksDB storage is empty, populate it from SQLite + let is_empty = self + .has_leaves() + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; + if !is_empty { + return load_smt(self); + } + + warn!(target: COMPONENT, "RocksDB account tree storage is empty, populating from SQLite"); + let account_data = db.select_all_account_commitments().await?; + let smt_entries = account_data + .into_iter() + .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); + LargeSmt::with_entries(self, smt_entries).map_err(large_smt_error_to_init_error) } async fn load_nullifier_tree( self, - _db: &mut Db, + db: &mut Db, ) -> Result>, StateInitializationError> { - let smt = load_smt(self)?; - Ok(NullifierTree::new_unchecked(smt)) + // If RocksDB storage is empty, populate it from SQLite + let is_empty = self + .has_leaves() + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; + if !is_empty { + let smt = load_smt(self)?; + return Ok(NullifierTree::new_unchecked(smt)); + } + + warn!(target: COMPONENT, "RocksDB nullifier tree storage is empty, populating from SQLite"); + let nullifiers = db.select_all_nullifiers().await?; + let entries = nullifiers.into_iter().map(|info| (info.nullifier, info.block_num)); + NullifierTree::with_storage_from_entries(self, entries) + .map_err(StateInitializationError::FailedToCreateNullifierTree) } } From 90c122f5d38061a5b4023df2468758af39ed488c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 11:43:15 +0100 Subject: [PATCH 14/23] fix --- crates/store/src/state.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 284f9f4bb..2c983f79f 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -29,23 +29,21 @@ use miden_node_utils::formatting::format_array; use miden_protocol::Word; use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::account::{AccountId, StorageSlotContent}; -#[cfg(not(feature = "rocksdb"))] -use miden_protocol::block::account_tree::account_id_to_smt_key; -use miden_protocol::block::account_tree::{AccountTree, AccountWitness}; +use miden_protocol::block::account_tree::{account_id_to_smt_key, AccountTree, AccountWitness}; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; #[cfg(not(feature = "rocksdb"))] use miden_protocol::crypto::merkle::smt::MemoryStorage; #[cfg(feature = "rocksdb")] -use tracing::warn; -#[cfg(feature = "rocksdb")] use miden_protocol::crypto::merkle::smt::RocksDbConfig; use miden_protocol::crypto::merkle::smt::{LargeSmt, LargeSmtError, SmtProof, SmtStorage}; use miden_protocol::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::{OutputNote, PartialBlockchain}; use miden_protocol::utils::Serializable; use tokio::sync::{Mutex, RwLock, oneshot}; +#[cfg(feature = "rocksdb")] +use tracing::warn; use tracing::{info, info_span, instrument}; use crate::accounts::{AccountTreeWithHistory, HistoricalError}; From a29ed14b20cfe8fd548e8da075f70ac80d654668 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 11:46:17 +0100 Subject: [PATCH 15/23] fix --- crates/store/src/state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 2c983f79f..cbdd4d553 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -28,8 +28,8 @@ use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; use miden_protocol::Word; use miden_protocol::account::delta::AccountUpdateDetails; -use miden_protocol::account::{AccountId, StorageSlotContent}; -use miden_protocol::block::account_tree::{account_id_to_smt_key, AccountTree, AccountWitness}; +use miden_protocol::account::AccountId; +use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; From f2be23cdc78ed42ffc4d234d0484eef236c7a84f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 14:00:39 +0100 Subject: [PATCH 16/23] review --- .github/actions/install-rocksdb/action.yml | 2 +- .github/workflows/msrv.yml | 2 ++ .github/workflows/stress-test-check.yml | 2 ++ .github/workflows/test-beta.yml | 2 ++ crates/store/Cargo.toml | 3 +++ crates/store/src/state.rs | 17 ++++++++--------- 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/actions/install-rocksdb/action.yml b/.github/actions/install-rocksdb/action.yml index 62b32b198..c42cb9825 100644 --- a/.github/actions/install-rocksdb/action.yml +++ b/.github/actions/install-rocksdb/action.yml @@ -1,4 +1,4 @@ -name: "Install RocksDB" +name: "Install RocksDB dependencies" description: "Install dependencies for RocksDB compilation" runs: diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index a5f383dab..1f0f26d8c 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -24,6 +24,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner - name: Install RocksDB uses: ./.github/actions/install-rocksdb - name: Install dependencies diff --git a/.github/workflows/stress-test-check.yml b/.github/workflows/stress-test-check.yml index d543d8c83..94f3d9638 100644 --- a/.github/workflows/stress-test-check.yml +++ b/.github/workflows/stress-test-check.yml @@ -27,6 +27,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@main + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner - name: Install RocksDB uses: ./.github/actions/install-rocksdb - name: Rustup diff --git a/.github/workflows/test-beta.yml b/.github/workflows/test-beta.yml index 146e68751..7d3a84ccf 100644 --- a/.github/workflows/test-beta.yml +++ b/.github/workflows/test-beta.yml @@ -16,6 +16,8 @@ jobs: - uses: actions/checkout@v4 with: ref: 'next' + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner - name: Install RocksDB uses: ./.github/actions/install-rocksdb - name: Rustup diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index ff737ffe7..4721ff69c 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -66,4 +66,7 @@ name = "account_tree" required-features = ["rocksdb"] [package.metadata.cargo-machete] +# This is an indirect dependency for which we need to enable optimisations +# via feature flags. Because we don't use it directly in code, machete +# identifies it as unused. ignored = ["miden-crypto"] diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index cbdd4d553..0d4bd0ad0 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -8,8 +8,6 @@ use std::ops::RangeInclusive; use std::path::Path; use std::sync::Arc; -#[cfg(feature = "rocksdb")] -use miden_crypto::merkle::smt::RocksDbStorage; use miden_node_proto::domain::account::{ AccountDetailRequest, AccountDetails, @@ -27,24 +25,25 @@ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; use miden_protocol::Word; -use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::account::AccountId; +use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; #[cfg(not(feature = "rocksdb"))] use miden_protocol::crypto::merkle::smt::MemoryStorage; -#[cfg(feature = "rocksdb")] -use miden_protocol::crypto::merkle::smt::RocksDbConfig; use miden_protocol::crypto::merkle::smt::{LargeSmt, LargeSmtError, SmtProof, SmtStorage}; use miden_protocol::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::{OutputNote, PartialBlockchain}; use miden_protocol::utils::Serializable; use tokio::sync::{Mutex, RwLock, oneshot}; -#[cfg(feature = "rocksdb")] -use tracing::warn; use tracing::{info, info_span, instrument}; +#[cfg(feature = "rocksdb")] +use { + miden_crypto::merkle::smt::RocksDbStorage, + miden_protocol::crypto::merkle::smt::RocksDbConfig, +}; use crate::accounts::{AccountTreeWithHistory, HistoricalError}; use crate::blocks::BlockStore; @@ -182,7 +181,7 @@ impl StorageLoader for RocksDbStorage { return load_smt(self); } - warn!(target: COMPONENT, "RocksDB account tree storage is empty, populating from SQLite"); + info!(target: COMPONENT, "RocksDB account tree storage is empty, populating from SQLite"); let account_data = db.select_all_account_commitments().await?; let smt_entries = account_data .into_iter() @@ -203,7 +202,7 @@ impl StorageLoader for RocksDbStorage { return Ok(NullifierTree::new_unchecked(smt)); } - warn!(target: COMPONENT, "RocksDB nullifier tree storage is empty, populating from SQLite"); + info!(target: COMPONENT, "RocksDB nullifier tree storage is empty, populating from SQLite"); let nullifiers = db.select_all_nullifiers().await?; let entries = nullifiers.into_iter().map(|info| (info.nullifier, info.block_num)); NullifierTree::with_storage_from_entries(self, entries) From d8689654057e9d586ea9b4b1c25367152ef02c36 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 14:15:27 +0100 Subject: [PATCH 17/23] more --- crates/store/src/state.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 0d4bd0ad0..36d85a532 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -90,7 +90,7 @@ pub type TreeStorage = RocksDbStorage; pub type TreeStorage = MemoryStorage; /// Converts a `LargeSmtError` into a `StateInitializationError`. -fn large_smt_error_to_init_error(e: LargeSmtError) -> StateInitializationError { +fn account_tree_large_smt_error_to_init_error(e: LargeSmtError) -> StateInitializationError { match e { LargeSmtError::Merkle(merkle_error) => { StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) @@ -104,7 +104,7 @@ fn large_smt_error_to_init_error(e: LargeSmtError) -> StateInitializationError { /// Loads an SMT from persistent storage. #[cfg(feature = "rocksdb")] fn load_smt(storage: S) -> Result, StateInitializationError> { - LargeSmt::new(storage).map_err(large_smt_error_to_init_error) + LargeSmt::new(storage).map_err(account_tree_large_smt_error_to_init_error) } /// Trait for loading trees from storage. @@ -145,7 +145,8 @@ impl StorageLoader for MemoryStorage { let smt_entries = account_data .into_iter() .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); - LargeSmt::with_entries(self, smt_entries).map_err(large_smt_error_to_init_error) + LargeSmt::with_entries(self, smt_entries) + .map_err(account_tree_large_smt_error_to_init_error) } async fn load_nullifier_tree( @@ -186,7 +187,8 @@ impl StorageLoader for RocksDbStorage { let smt_entries = account_data .into_iter() .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); - LargeSmt::with_entries(self, smt_entries).map_err(large_smt_error_to_init_error) + LargeSmt::with_entries(self, smt_entries) + .map_err(account_tree_large_smt_error_to_init_error) } async fn load_nullifier_tree( @@ -196,7 +198,7 @@ impl StorageLoader for RocksDbStorage { // If RocksDB storage is empty, populate it from SQLite let is_empty = self .has_leaves() - .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; + .map_err(|e| StateInitializationError::NullifierTreeIoError(e.to_string()))?; if !is_empty { let smt = load_smt(self)?; return Ok(NullifierTree::new_unchecked(smt)); From d2e5a52a82fe30939a7139509e1618ad72c1e6da Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 14:25:28 +0100 Subject: [PATCH 18/23] taplo --- crates/store/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 4721ff69c..062c4dde3 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -61,8 +61,8 @@ default = ["rocksdb"] rocksdb = ["miden-crypto/rocksdb"] [[bench]] -harness = false -name = "account_tree" +harness = false +name = "account_tree" required-features = ["rocksdb"] [package.metadata.cargo-machete] From b8fa48f5d6c03bd341c6356d1717e86c51dee20a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 15:37:29 +0100 Subject: [PATCH 19/23] yesyes, booleans --- crates/store/src/state.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 36d85a532..2ff1f887b 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -174,11 +174,11 @@ impl StorageLoader for RocksDbStorage { self, db: &mut Db, ) -> Result, StateInitializationError> { - // If RocksDB storage is empty, populate it from SQLite - let is_empty = self + // If RocksDB storage has data, load from it directly + let has_data = self .has_leaves() .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; - if !is_empty { + if has_data { return load_smt(self); } @@ -195,11 +195,11 @@ impl StorageLoader for RocksDbStorage { self, db: &mut Db, ) -> Result>, StateInitializationError> { - // If RocksDB storage is empty, populate it from SQLite - let is_empty = self + // If RocksDB storage has data, load from it directly + let has_data = self .has_leaves() .map_err(|e| StateInitializationError::NullifierTreeIoError(e.to_string()))?; - if !is_empty { + if has_data { let smt = load_smt(self)?; return Ok(NullifierTree::new_unchecked(smt)); } From 35976f3d5aee0ce1702918607d2cdcdff4d0c34b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 16:14:02 +0100 Subject: [PATCH 20/23] fix --- crates/block-producer/src/server/tests.rs | 31 ++++++++-- crates/rpc/src/tests.rs | 75 ++++++++++++++--------- 2 files changed, 73 insertions(+), 33 deletions(-) diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index cbfd27fe0..1aa7c023d 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -124,6 +124,27 @@ async fn block_producer_startup_is_robust_to_network_failures() { assert!(response.is_err()); // test: restart the store and request should succeed + let store_runtime = restart_store(store_addr, data_directory.path()).await; + let response = send_request(block_producer_client.clone(), 2).await; + assert!(response.is_ok()); + + // Shutdown the store before data_directory is dropped to allow RocksDB to flush properly + shutdown_store(store_runtime).await; +} + +/// Shuts down the store runtime properly to allow RocksDB to flush before the temp directory is +/// deleted. +async fn shutdown_store(store_runtime: runtime::Runtime) { + task::spawn_blocking(move || store_runtime.shutdown_timeout(Duration::from_millis(500))) + .await + .expect("shutdown should complete"); +} + +/// Restarts a store using an existing data directory. Returns the runtime handle for shutdown. +async fn restart_store( + store_addr: std::net::SocketAddr, + data_directory: &std::path::Path, +) -> runtime::Runtime { let rpc_listener = TcpListener::bind("127.0.0.1:0").await.expect("store should bind the RPC port"); let ntx_builder_listener = TcpListener::bind("127.0.0.1:0") @@ -132,19 +153,21 @@ async fn block_producer_startup_is_robust_to_network_failures() { let block_producer_listener = TcpListener::bind(store_addr) .await .expect("store should bind the block-producer port"); - task::spawn(async move { + let dir = data_directory.to_path_buf(); + let store_runtime = + runtime::Builder::new_multi_thread().enable_time().enable_io().build().unwrap(); + store_runtime.spawn(async move { Store { rpc_listener, ntx_builder_listener, block_producer_listener, - data_directory: data_directory.path().to_path_buf(), + data_directory: dir, } .serve() .await .expect("store should start serving"); }); - let response = send_request(block_producer_client.clone(), 2).await; - assert!(response.is_ok()); + store_runtime } /// Creates a dummy transaction and submits it to the block producer. diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index 263ef9bfb..e88ee4096 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -64,7 +64,7 @@ async fn rpc_server_accepts_requests_without_accept_header() { assert!(response.is_ok()); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } #[tokio::test] @@ -80,7 +80,7 @@ async fn rpc_server_accepts_requests_with_accept_header() { assert!(response.is_ok()); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } #[tokio::test] @@ -113,7 +113,7 @@ async fn rpc_server_rejects_requests_with_accept_header_invalid_version() { assert!(response.as_ref().err().unwrap().message().contains("server does not support"),); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } } @@ -137,34 +137,17 @@ async fn rpc_startup_is_robust_to_network_failures() { assert!(response.unwrap().into_inner().block_header.is_some()); // Test: shutdown the store and should fail - // Use spawn_blocking because shutdown_timeout blocks and can't run in async context - task::spawn_blocking(move || store_runtime.shutdown_timeout(Duration::from_millis(500))) - .await - .expect("shutdown should complete"); + shutdown_store(store_runtime).await; let response = send_request(&mut rpc_client).await; assert!(response.is_err()); // Test: restart the store and request should succeed - let rpc_listener = TcpListener::bind(store_addr).await.expect("Failed to bind store"); - let ntx_builder_listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("Failed to bind store ntx-builder gRPC endpoint"); - let block_producer_listener = - TcpListener::bind("127.0.0.1:0").await.expect("store should bind a port"); - task::spawn(async move { - Store { - rpc_listener, - ntx_builder_listener, - block_producer_listener, - data_directory: data_directory.path().to_path_buf(), - grpc_timeout: Duration::from_secs(10), - } - .serve() - .await - .expect("store should start serving"); - }); + let store_runtime = restart_store(store_addr, data_directory.path()).await; let response = send_request(&mut rpc_client).await; assert_eq!(response.unwrap().into_inner().block_header.unwrap().block_num, 0); + + // Shutdown the store before data_directory is dropped to allow RocksDB to flush properly + shutdown_store(store_runtime).await; } #[tokio::test] @@ -207,7 +190,7 @@ async fn rpc_server_has_web_support() { assert!(headers.get("access-control-allow-credentials").is_some()); assert!(headers.get("access-control-expose-headers").is_some()); assert!(headers.get("vary").is_some()); - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } #[tokio::test] @@ -293,7 +276,7 @@ async fn rpc_server_rejects_proven_transactions_with_invalid_commitment() { ); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } #[tokio::test] @@ -366,7 +349,7 @@ async fn rpc_server_rejects_tx_submissions_without_genesis() { ); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } /// Sends an arbitrary / irrelevant request to the RPC. @@ -472,6 +455,40 @@ async fn start_store(store_addr: SocketAddr) -> (Runtime, TempDir, Word) { ) } +/// Shuts down the store runtime properly to allow `RocksDB` to flush before the temp directory is +/// deleted. +async fn shutdown_store(store_runtime: Runtime) { + task::spawn_blocking(move || store_runtime.shutdown_timeout(Duration::from_millis(500))) + .await + .expect("shutdown should complete"); +} + +/// Restarts a store using an existing data directory. Returns the runtime handle for shutdown. +async fn restart_store(store_addr: SocketAddr, data_directory: &std::path::Path) -> Runtime { + let rpc_listener = TcpListener::bind(store_addr).await.expect("Failed to bind store"); + let ntx_builder_listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("Failed to bind store ntx-builder gRPC endpoint"); + let block_producer_listener = + TcpListener::bind("127.0.0.1:0").await.expect("store should bind a port"); + let dir = data_directory.to_path_buf(); + let store_runtime = + runtime::Builder::new_multi_thread().enable_time().enable_io().build().unwrap(); + store_runtime.spawn(async move { + Store { + rpc_listener, + ntx_builder_listener, + block_producer_listener, + data_directory: dir, + grpc_timeout: Duration::from_secs(10), + } + .serve() + .await + .expect("store should start serving"); + }); + store_runtime +} + #[tokio::test] async fn get_limits_endpoint() { // Start the RPC and store @@ -524,5 +541,5 @@ async fn get_limits_endpoint() { ); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } From 58fd77b34e0e70c439e66a54eccc295be48a10cb Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 17:05:37 +0100 Subject: [PATCH 21/23] 10m is not enough --- .github/workflows/stress-test-check.yml | 2 +- crates/block-producer/src/server/tests.rs | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/stress-test-check.yml b/.github/workflows/stress-test-check.yml index 94f3d9638..b41ad9283 100644 --- a/.github/workflows/stress-test-check.yml +++ b/.github/workflows/stress-test-check.yml @@ -24,7 +24,7 @@ jobs: stress-test-check: name: stress-test-check runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 20 steps: - uses: actions/checkout@main - name: Cleanup large tools for build space diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index 1aa7c023d..91de51ddc 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -114,10 +114,7 @@ async fn block_producer_startup_is_robust_to_network_failures() { assert!(response.is_ok()); // kill the store - // Use spawn_blocking because shutdown_timeout blocks and can't run in async context - task::spawn_blocking(move || store_runtime.shutdown_timeout(Duration::from_millis(500))) - .await - .expect("shutdown should complete"); + shutdown_store(store_runtime).await; // test: request against block-producer api should fail immediately let response = send_request(block_producer_client.clone(), 1).await; From 46461a86d375e92b525417e5865ce191a2edc4ef Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 17:26:31 +0100 Subject: [PATCH 22/23] proper runners for heavy duty tasks --- .github/workflows/msrv.yml | 2 +- .github/workflows/stress-test-check.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index 1f0f26d8c..fb261e73e 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -21,7 +21,7 @@ env: jobs: # Check MSRV (aka `rust-version`) in `Cargo.toml` is valid for workspace members msrv: - runs-on: ubuntu-latest + runs-on: Linux-ARM64-Runner steps: - uses: actions/checkout@v4 - name: Cleanup large tools for build space diff --git a/.github/workflows/stress-test-check.yml b/.github/workflows/stress-test-check.yml index e8ad5d60e..383440b9e 100644 --- a/.github/workflows/stress-test-check.yml +++ b/.github/workflows/stress-test-check.yml @@ -23,7 +23,7 @@ env: jobs: stress-test-check: name: stress-test-check - runs-on: ubuntu-24.04 + runs-on: Linux-ARM64-Runner timeout-minutes: 20 steps: - uses: actions/checkout@main From 6357928be4a05b9e59c689d6f64e74dc69967239 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 16 Jan 2026 17:31:56 +0100 Subject: [PATCH 23/23] accidentally readded, removed in 50680fe --- .github/workflows/msrv.yml | 41 -------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 .github/workflows/msrv.yml diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml deleted file mode 100644 index fb261e73e..000000000 --- a/.github/workflows/msrv.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Check MSRV - -on: - push: - branches: [next] - pull_request: - types: [opened, reopened, synchronize] - -# Limits workflow concurrency to only the latest commit in the PR. -concurrency: - group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}" - cancel-in-progress: true - -permissions: - contents: read - -env: - # Reduce cache usage by removing debug information. - CARGO_PROFILE_DEV_DEBUG: 0 - -jobs: - # Check MSRV (aka `rust-version`) in `Cargo.toml` is valid for workspace members - msrv: - runs-on: Linux-ARM64-Runner - steps: - - uses: actions/checkout@v4 - - name: Cleanup large tools for build space - uses: ./.github/actions/cleanup-runner - - name: Install RocksDB - uses: ./.github/actions/install-rocksdb - - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y jq - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - with: - save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }} - - name: Install cargo-msrv - run: cargo install cargo-msrv - - name: Check MSRV for each workspace member - run: | - ./scripts/check-msrv.sh