diff --git a/.github/actions/install-rocksdb/action.yml b/.github/actions/install-rocksdb/action.yml new file mode 100644 index 000000000..c42cb9825 --- /dev/null +++ b/.github/actions/install-rocksdb/action.yml @@ -0,0 +1,14 @@ +name: "Install RocksDB dependencies" +description: "Install dependencies for RocksDB compilation" + +runs: + using: "composite" + steps: + - name: Install LLVM/Clang for RocksDB + shell: bash + run: | + set -eux + sudo apt-get update + # Install clang/llvm for bindgen (needed for FFI bindings). + # RocksDB is compiled from source by librocksdb-sys. + sudo apt-get install -y clang llvm-dev libclang-dev diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 09500e58f..d2dea8a72 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -51,6 +51,10 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@main + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: | rustup update --no-self-update @@ -87,6 +91,10 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@main + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/publish-debian-all.yml b/.github/workflows/publish-debian-all.yml index 6a8a8f138..a6d63d503 100644 --- a/.github/workflows/publish-debian-all.yml +++ b/.github/workflows/publish-debian-all.yml @@ -31,6 +31,8 @@ jobs: uses: actions/checkout@main with: fetch-depth: 0 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Build and Publish Node uses: ./.github/actions/debian with: diff --git a/.github/workflows/publish-debian.yml b/.github/workflows/publish-debian.yml index f115f5eef..81e8d7447 100644 --- a/.github/workflows/publish-debian.yml +++ b/.github/workflows/publish-debian.yml @@ -60,6 +60,9 @@ jobs: with: fetch-depth: 0 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Build and Publish Packages uses: ./.github/actions/debian with: diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml index 7b059d518..fe6b15e87 100644 --- a/.github/workflows/publish-dry-run.yml +++ b/.github/workflows/publish-dry-run.yml @@ -23,6 +23,8 @@ jobs: fetch-depth: 0 - name: Cleanup large tools for build space uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y jq - name: Update Rust toolchain diff --git a/.github/workflows/publish-main.yml b/.github/workflows/publish-main.yml index 748f2291a..fcaab36a8 100644 --- a/.github/workflows/publish-main.yml +++ b/.github/workflows/publish-main.yml @@ -18,6 +18,8 @@ jobs: with: fetch-depth: 0 ref: main + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb # Ensure the release tag refers to the latest commit on main. # Compare the commit SHA that triggered the workflow with the HEAD of the branch we just # checked out (main). diff --git a/.github/workflows/stress-test-check.yml b/.github/workflows/stress-test-check.yml index 605cf7b5b..383440b9e 100644 --- a/.github/workflows/stress-test-check.yml +++ b/.github/workflows/stress-test-check.yml @@ -23,10 +23,14 @@ env: jobs: stress-test-check: name: stress-test-check - runs-on: ubuntu-24.04 - timeout-minutes: 10 + runs-on: Linux-ARM64-Runner + timeout-minutes: 20 steps: - uses: actions/checkout@main + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/test-beta.yml b/.github/workflows/test-beta.yml index 63aae6775..07b9705fd 100644 --- a/.github/workflows/test-beta.yml +++ b/.github/workflows/test-beta.yml @@ -16,6 +16,10 @@ jobs: - uses: actions/checkout@v4 with: ref: 'next' + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: rustup install beta && rustup default beta - uses: taiki-e/install-action@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c86c0f25f..7760225a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,10 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@main + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index abcd9359a..bbd5407e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,6 +115,7 @@ - Add optional `TransactionInputs` field to `SubmitProvenTransaction` endpoint for transaction re-execution (#[1278](https://github.com/0xMiden/miden-node/pull/1278)). - Added `validator` crate with initial protobuf, gRPC server, and sub-command (#[1293](https://github.com/0xMiden/miden-node/pull/1293)). - [BREAKING] Added `AccountTreeWithHistory` and integrate historical queries into `GetAccountProof` ([#1292](https://github.com/0xMiden/miden-node/pull/1292)). +- [BREAKING] Added `rocksdb` feature to enable rocksdb backends of `LargeSmt` ([#1326](https://github.com/0xMiden/miden-node/pull/1326)). - [BREAKING] Handle past/historical `AccountProof` requests ([#1333](https://github.com/0xMiden/miden-node/pull/1333)). - Implement `DataStore::get_note_script()` for `NtxDataStore` (#[1332](https://github.com/0xMiden/miden-node/pull/1332)). - Started validating notes by their commitment instead of ID before entering the mempool ([#1338](https://github.com/0xMiden/miden-node/pull/1338)). diff --git a/Cargo.lock b/Cargo.lock index 1ad02438b..aa8a55777 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,6 +342,24 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.114", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -446,6 +464,16 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "camino" version = "1.2.2" @@ -496,6 +524,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cf-rustracing" version = "1.2.1" @@ -605,6 +642,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "3.2.25" @@ -2229,12 +2277,36 @@ version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "librocksdb-sys" +version = "0.17.3+10.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", +] + [[package]] name = "libsqlite3-sys" version = "0.35.0" @@ -2255,6 +2327,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libz-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2367,6 +2450,16 @@ version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2527,6 +2620,7 @@ dependencies = [ "rand_core 0.9.5", "rand_hc", "rayon", + "rocksdb", "sha2", "sha3", "subtle", @@ -2824,6 +2918,7 @@ dependencies = [ "fs-err", "hex", "indexmap 2.13.0", + "miden-crypto", "miden-node-proto", "miden-node-proto-build", "miden-node-test-macro", @@ -3248,6 +3343,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3348,6 +3449,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4626,6 +4737,16 @@ dependencies = [ "serde", ] +[[package]] +name = "rocksdb" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rstest" version = "0.26.1" @@ -4671,6 +4792,12 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 53e5182bb..6acc89250 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,8 @@ miden-tx = { branch = "next", default-features = false, git = "http miden-tx-batch-prover = { branch = "next", git = "https://github.com/0xMiden/miden-base.git" } # Other miden dependencies. These should align with those expected by miden-base. -miden-air = { features = ["std", "testing"], version = "0.20" } +miden-air = { features = ["std", "testing"], version = "0.20" } +miden-crypto = { default-features = false, version = "0.19" } # External dependencies anyhow = { version = "1.0" } diff --git a/bin/node/Dockerfile b/bin/node/Dockerfile index 3becd3ded..832b0bb8d 100644 --- a/bin/node/Dockerfile +++ b/bin/node/Dockerfile @@ -1,8 +1,9 @@ FROM rust:1.90-slim-bullseye AS builder +# Install build dependencies. RocksDB is compiled from source by librocksdb-sys. RUN apt-get update && \ apt-get -y upgrade && \ - apt-get install -y llvm clang bindgen pkg-config libssl-dev libsqlite3-dev ca-certificates && \ + apt-get install -y llvm clang libclang-dev pkg-config libssl-dev libsqlite3-dev ca-certificates && \ rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index cbfd27fe0..91de51ddc 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -114,16 +114,34 @@ async fn block_producer_startup_is_robust_to_network_failures() { assert!(response.is_ok()); // kill the store - // Use spawn_blocking because shutdown_timeout blocks and can't run in async context - task::spawn_blocking(move || store_runtime.shutdown_timeout(Duration::from_millis(500))) - .await - .expect("shutdown should complete"); + shutdown_store(store_runtime).await; // test: request against block-producer api should fail immediately let response = send_request(block_producer_client.clone(), 1).await; assert!(response.is_err()); // test: restart the store and request should succeed + let store_runtime = restart_store(store_addr, data_directory.path()).await; + let response = send_request(block_producer_client.clone(), 2).await; + assert!(response.is_ok()); + + // Shutdown the store before data_directory is dropped to allow RocksDB to flush properly + shutdown_store(store_runtime).await; +} + +/// Shuts down the store runtime properly to allow RocksDB to flush before the temp directory is +/// deleted. +async fn shutdown_store(store_runtime: runtime::Runtime) { + task::spawn_blocking(move || store_runtime.shutdown_timeout(Duration::from_millis(500))) + .await + .expect("shutdown should complete"); +} + +/// Restarts a store using an existing data directory. Returns the runtime handle for shutdown. +async fn restart_store( + store_addr: std::net::SocketAddr, + data_directory: &std::path::Path, +) -> runtime::Runtime { let rpc_listener = TcpListener::bind("127.0.0.1:0").await.expect("store should bind the RPC port"); let ntx_builder_listener = TcpListener::bind("127.0.0.1:0") @@ -132,19 +150,21 @@ async fn block_producer_startup_is_robust_to_network_failures() { let block_producer_listener = TcpListener::bind(store_addr) .await .expect("store should bind the block-producer port"); - task::spawn(async move { + let dir = data_directory.to_path_buf(); + let store_runtime = + runtime::Builder::new_multi_thread().enable_time().enable_io().build().unwrap(); + store_runtime.spawn(async move { Store { rpc_listener, ntx_builder_listener, block_producer_listener, - data_directory: data_directory.path().to_path_buf(), + data_directory: dir, } .serve() .await .expect("store should start serving"); }); - let response = send_request(block_producer_client.clone(), 2).await; - assert!(response.is_ok()); + store_runtime } /// Creates a dummy transaction and submits it to the block producer. diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index 263ef9bfb..e88ee4096 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -64,7 +64,7 @@ async fn rpc_server_accepts_requests_without_accept_header() { assert!(response.is_ok()); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } #[tokio::test] @@ -80,7 +80,7 @@ async fn rpc_server_accepts_requests_with_accept_header() { assert!(response.is_ok()); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } #[tokio::test] @@ -113,7 +113,7 @@ async fn rpc_server_rejects_requests_with_accept_header_invalid_version() { assert!(response.as_ref().err().unwrap().message().contains("server does not support"),); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } } @@ -137,34 +137,17 @@ async fn rpc_startup_is_robust_to_network_failures() { assert!(response.unwrap().into_inner().block_header.is_some()); // Test: shutdown the store and should fail - // Use spawn_blocking because shutdown_timeout blocks and can't run in async context - task::spawn_blocking(move || store_runtime.shutdown_timeout(Duration::from_millis(500))) - .await - .expect("shutdown should complete"); + shutdown_store(store_runtime).await; let response = send_request(&mut rpc_client).await; assert!(response.is_err()); // Test: restart the store and request should succeed - let rpc_listener = TcpListener::bind(store_addr).await.expect("Failed to bind store"); - let ntx_builder_listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("Failed to bind store ntx-builder gRPC endpoint"); - let block_producer_listener = - TcpListener::bind("127.0.0.1:0").await.expect("store should bind a port"); - task::spawn(async move { - Store { - rpc_listener, - ntx_builder_listener, - block_producer_listener, - data_directory: data_directory.path().to_path_buf(), - grpc_timeout: Duration::from_secs(10), - } - .serve() - .await - .expect("store should start serving"); - }); + let store_runtime = restart_store(store_addr, data_directory.path()).await; let response = send_request(&mut rpc_client).await; assert_eq!(response.unwrap().into_inner().block_header.unwrap().block_num, 0); + + // Shutdown the store before data_directory is dropped to allow RocksDB to flush properly + shutdown_store(store_runtime).await; } #[tokio::test] @@ -207,7 +190,7 @@ async fn rpc_server_has_web_support() { assert!(headers.get("access-control-allow-credentials").is_some()); assert!(headers.get("access-control-expose-headers").is_some()); assert!(headers.get("vary").is_some()); - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } #[tokio::test] @@ -293,7 +276,7 @@ async fn rpc_server_rejects_proven_transactions_with_invalid_commitment() { ); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } #[tokio::test] @@ -366,7 +349,7 @@ async fn rpc_server_rejects_tx_submissions_without_genesis() { ); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } /// Sends an arbitrary / irrelevant request to the RPC. @@ -472,6 +455,40 @@ async fn start_store(store_addr: SocketAddr) -> (Runtime, TempDir, Word) { ) } +/// Shuts down the store runtime properly to allow `RocksDB` to flush before the temp directory is +/// deleted. +async fn shutdown_store(store_runtime: Runtime) { + task::spawn_blocking(move || store_runtime.shutdown_timeout(Duration::from_millis(500))) + .await + .expect("shutdown should complete"); +} + +/// Restarts a store using an existing data directory. Returns the runtime handle for shutdown. +async fn restart_store(store_addr: SocketAddr, data_directory: &std::path::Path) -> Runtime { + let rpc_listener = TcpListener::bind(store_addr).await.expect("Failed to bind store"); + let ntx_builder_listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("Failed to bind store ntx-builder gRPC endpoint"); + let block_producer_listener = + TcpListener::bind("127.0.0.1:0").await.expect("store should bind a port"); + let dir = data_directory.to_path_buf(); + let store_runtime = + runtime::Builder::new_multi_thread().enable_time().enable_io().build().unwrap(); + store_runtime.spawn(async move { + Store { + rpc_listener, + ntx_builder_listener, + block_producer_listener, + data_directory: dir, + grpc_timeout: Duration::from_secs(10), + } + .serve() + .await + .expect("store should start serving"); + }); + store_runtime +} + #[tokio::test] async fn get_limits_endpoint() { // Start the RPC and store @@ -524,5 +541,5 @@ async fn get_limits_endpoint() { ); // Shutdown to avoid runtime drop error. - store_runtime.shutdown_background(); + shutdown_store(store_runtime).await; } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 22037e4b9..062c4dde3 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -24,6 +24,7 @@ diesel_migrations = { features = ["sqlite"], version = "2.3" } fs-err = { workspace = true } hex = { version = "0.4" } indexmap = { workspace = true } +miden-crypto = { features = ["concurrent", "hashmaps"], workspace = true } miden-node-proto = { workspace = true } miden-node-proto-build = { features = ["internal"], workspace = true } miden-node-utils = { workspace = true } @@ -55,6 +56,17 @@ rand = { workspace = true } regex = { version = "1.11" } termtree = { version = "0.5" } +[features] +default = ["rocksdb"] +rocksdb = ["miden-crypto/rocksdb"] + [[bench]] -harness = false -name = "account_tree_historical" +harness = false +name = "account_tree" +required-features = ["rocksdb"] + +[package.metadata.cargo-machete] +# This is an indirect dependency for which we need to enable optimisations +# via feature flags. Because we don't use it directly in code, machete +# identifies it as unused. +ignored = ["miden-crypto"] diff --git a/crates/store/README.md b/crates/store/README.md index e3f9a8dde..57c002fe5 100644 --- a/crates/store/README.md +++ b/crates/store/README.md @@ -7,6 +7,35 @@ operator must take care that the store's API endpoint is **only** exposed to the For more information on the installation and operation of this component, please see the [node's readme](/README.md). +## RocksDB Feature + +The `rocksdb` feature (enabled by default) provides disk-backed storage via RocksDB for `LargeSmt`. Building _requires_ LLVM/Clang for `bindgen`. + +### Using System Libraries + +To avoid compiling RocksDB from source and safe yourself some time, use system libraries: + +```bash +# Install system RocksDB +# (Ubuntu/Debian) +#sudo apt-get install librocksdb-dev clang llvm-dev libclang-dev +# (Fedora) +#sudo dnf install rocksdb rocksdb-devel llvm19 clang19 + +# Set environment variables to use system library +export ROCKSDB_LIB_DIR=/usr/lib +export ROCKSDB_INCLUDE_DIR=/usr/include +# export ROCKSDB_STATIC=1 (optional) +# (Ubuntu/Debian) +#export LIBCLANG_PATH=/usr/lib/llvm-14/lib +# (Fedora) +#export LIBCLANG_PATH=/usr/lib64/llvm19/lib +``` + +### Building from Source + +Without the environment variables above, `librocksdb-sys` compiles RocksDB from source, which requires a C/C++ toolchain. + ## API overview The full gRPC API can be found [here](../../proto/proto/store.proto). diff --git a/crates/store/benches/account_tree_historical.rs b/crates/store/benches/account_tree.rs similarity index 81% rename from crates/store/benches/account_tree_historical.rs rename to crates/store/benches/account_tree.rs index ba7a5c2cc..8c3f1009e 100644 --- a/crates/store/benches/account_tree_historical.rs +++ b/crates/store/benches/account_tree.rs @@ -1,22 +1,44 @@ use std::hint::black_box; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use miden_crypto::merkle::smt::{RocksDbConfig, RocksDbStorage}; use miden_node_store::AccountTreeWithHistory; use miden_protocol::Word; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; use miden_protocol::block::account_tree::{AccountTree, account_id_to_smt_key}; use miden_protocol::crypto::hash::rpo::Rpo256; -use miden_protocol::crypto::merkle::smt::{LargeSmt, MemoryStorage}; +use miden_protocol::crypto::merkle::smt::LargeSmt; use miden_protocol::testing::account_id::AccountIdBuilder; +/// Counter for creating unique `RocksDB` directories during benchmarking. +static DB_COUNTER: AtomicUsize = AtomicUsize::new(0); + // HELPER FUNCTIONS // ================================================================================================ -/// Creates a storage backend for a `LargeSmt`. -fn setup_storage() -> MemoryStorage { - // TODO migrate to RocksDB for persistence to gain meaningful numbers - MemoryStorage::default() +/// Returns the default base path for `RocksDB` benchmark storage. +fn default_storage_path() -> std::path::PathBuf { + std::path::PathBuf::from("target/bench_rocksdb") +} + +/// Creates a `RocksDB` storage instance for benchmarking. +/// +/// # Arguments +/// * `base_path` - Base directory for `RocksDB` storage. Each call creates a unique subdirectory. +fn setup_storage(base_path: &Path) -> RocksDbStorage { + let counter = DB_COUNTER.fetch_add(1, Ordering::SeqCst); + let db_path = base_path.join(format!("bench_rocksdb_{counter}")); + + // Clean up the directory if it exists + if db_path.exists() { + fs_err::remove_dir_all(&db_path).ok(); + } + fs_err::create_dir_all(&db_path).expect("Failed to create storage directory"); + + RocksDbStorage::open(RocksDbConfig::new(db_path)).expect("RocksDB failed to open file") } /// Generates a deterministic word from a seed. @@ -47,7 +69,8 @@ fn generate_account_id(seed: &mut [u8; 32]) -> AccountId { /// Sets up a vanilla `AccountTree` with specified number of accounts. fn setup_vanilla_account_tree( num_accounts: usize, -) -> (AccountTree>, Vec) { + base_path: &Path, +) -> (AccountTree>, Vec) { let mut seed = [0u8; 32]; let mut account_ids = Vec::new(); let mut entries = Vec::new(); @@ -59,7 +82,7 @@ fn setup_vanilla_account_tree( entries.push((account_id_to_smt_key(account_id), commitment)); } - let storage = setup_storage(); + let storage = setup_storage(base_path); let smt = LargeSmt::with_entries(storage, entries).expect("Failed to create LargeSmt from entries"); let tree = AccountTree::new(smt).expect("Failed to create AccountTree"); @@ -70,9 +93,10 @@ fn setup_vanilla_account_tree( fn setup_account_tree_with_history( num_accounts: usize, num_blocks: usize, -) -> (AccountTreeWithHistory, Vec) { + base_path: &Path, +) -> (AccountTreeWithHistory, Vec) { let mut seed = [0u8; 32]; - let storage = setup_storage(); + let storage = setup_storage(base_path); let smt = LargeSmt::with_entries(storage, std::iter::empty()) .expect("Failed to create empty LargeSmt"); let account_tree = AccountTree::new(smt).expect("Failed to create AccountTree"); @@ -104,11 +128,12 @@ fn setup_account_tree_with_history( /// This provides a baseline for comparison with historical access operations. fn bench_vanilla_access(c: &mut Criterion) { let mut group = c.benchmark_group("account_tree_vanilla_access"); + let base_path = default_storage_path(); let account_counts = [1, 10, 50, 100, 500, 1000]; for &num_accounts in &account_counts { - let (tree, account_ids) = setup_vanilla_account_tree(num_accounts); + let (tree, account_ids) = setup_vanilla_account_tree(num_accounts, &base_path); group.bench_function(BenchmarkId::new("vanilla", num_accounts), |b| { let test_account = *account_ids.first().unwrap(); @@ -125,6 +150,7 @@ fn bench_vanilla_access(c: &mut Criterion) { /// This provides a baseline for comparison with history-tracking insertion. fn bench_vanilla_insertion(c: &mut Criterion) { let mut group = c.benchmark_group("account_tree_insertion"); + let base_path = default_storage_path(); let account_counts = [1, 10, 50, 100, 500]; @@ -132,7 +158,7 @@ fn bench_vanilla_insertion(c: &mut Criterion) { group.bench_function(BenchmarkId::new("vanilla", num_accounts), |b| { b.iter(|| { let mut seed = [0u8; 32]; - let storage = setup_storage(); + let storage = setup_storage(&base_path); let smt = LargeSmt::with_entries(storage, std::iter::empty()) .expect("Failed to create empty LargeSmt"); let mut tree = AccountTree::new(smt).expect("Failed to create AccountTree"); @@ -158,18 +184,19 @@ fn bench_vanilla_insertion(c: &mut Criterion) { /// Benchmarks historical access at different depths and account counts. fn bench_historical_access(c: &mut Criterion) { let mut group = c.benchmark_group("account_tree_historical_access"); + let base_path = default_storage_path(); let account_counts = [10, 100, 500, 2500]; let block_depths = [0, 5, 10, 20, 32]; for &num_accounts in &account_counts { for &block_depth in &block_depths { - if block_depth > AccountTreeWithHistory::::MAX_HISTORY { + if block_depth > AccountTreeWithHistory::::MAX_HISTORY { continue; } let (tree_hist, account_ids) = - setup_account_tree_with_history(num_accounts, block_depth + 1); + setup_account_tree_with_history(num_accounts, block_depth + 1, &base_path); let current_block = tree_hist.block_number_latest(); let target_block = current_block .checked_sub(u32::try_from(block_depth).unwrap()) @@ -197,6 +224,7 @@ fn bench_historical_access(c: &mut Criterion) { /// Benchmarks insertion performance with history tracking at different account counts. fn bench_insertion_with_history(c: &mut Criterion) { let mut group = c.benchmark_group("account_tree_insertion"); + let base_path = default_storage_path(); let account_counts = [1, 10, 50, 100, 500, 2500]; @@ -204,7 +232,7 @@ fn bench_insertion_with_history(c: &mut Criterion) { group.bench_function(BenchmarkId::new("with_history", num_accounts), |b| { b.iter(|| { let mut seed = [0u8; 32]; - let storage = setup_storage(); + let storage = setup_storage(&base_path); let smt = LargeSmt::with_entries(storage, std::iter::empty()) .expect("Failed to create empty LargeSmt"); let account_tree = AccountTree::new(smt).expect("Failed to create AccountTree"); diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index c0a37be32..7fa5be8e7 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -29,6 +29,10 @@ mod tests; /// Convenience for an in-memory-only account tree. pub type InMemoryAccountTree = AccountTree>; +#[cfg(feature = "rocksdb")] +/// Convenience for a persistent account tree. +pub type PersistentAccountTree = AccountTree>; + // HISTORICAL ERROR TYPES // ================================================================================================ diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 5a9dc5ee2..1d345dcf0 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -7,6 +7,8 @@ mod inner_forest; mod server; pub mod state; +#[cfg(feature = "rocksdb")] +pub use accounts::PersistentAccountTree; pub use accounts::{AccountTreeWithHistory, HistoricalError, InMemoryAccountTree}; pub use genesis::GenesisState; pub use server::{DataDirectory, Store}; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index d9594a87c..2ff1f887b 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -31,18 +31,19 @@ use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_i use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; -use miden_protocol::crypto::merkle::smt::{ - LargeSmt, - LargeSmtError, - MemoryStorage, - SmtProof, - SmtStorage, -}; +#[cfg(not(feature = "rocksdb"))] +use miden_protocol::crypto::merkle::smt::MemoryStorage; +use miden_protocol::crypto::merkle::smt::{LargeSmt, LargeSmtError, SmtProof, SmtStorage}; use miden_protocol::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::{OutputNote, PartialBlockchain}; use miden_protocol::utils::Serializable; use tokio::sync::{Mutex, RwLock, oneshot}; use tracing::{info, info_span, instrument}; +#[cfg(feature = "rocksdb")] +use { + miden_crypto::merkle::smt::RocksDbStorage, + miden_protocol::crypto::merkle::smt::RocksDbConfig, +}; use crate::accounts::{AccountTreeWithHistory, HistoricalError}; use crate::blocks::BlockStore; @@ -82,8 +83,137 @@ pub struct TransactionInputs { pub new_account_id_prefix_is_unique: Option, } +/// The storage backend for trees. +#[cfg(feature = "rocksdb")] +pub type TreeStorage = RocksDbStorage; +#[cfg(not(feature = "rocksdb"))] +pub type TreeStorage = MemoryStorage; + +/// Converts a `LargeSmtError` into a `StateInitializationError`. +fn account_tree_large_smt_error_to_init_error(e: LargeSmtError) -> StateInitializationError { + match e { + LargeSmtError::Merkle(merkle_error) => { + StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) + }, + LargeSmtError::Storage(err) => { + StateInitializationError::AccountTreeIoError(err.as_report()) + }, + } +} + +/// Loads an SMT from persistent storage. +#[cfg(feature = "rocksdb")] +fn load_smt(storage: S) -> Result, StateInitializationError> { + LargeSmt::new(storage).map_err(account_tree_large_smt_error_to_init_error) +} + +/// Trait for loading trees from storage. +/// +/// For `MemoryStorage`, the tree is rebuilt from database entries on each startup. +/// For `RocksDbStorage`, the tree is loaded directly from disk (much faster for large trees). +// TODO handle on disk rocksdb storage file being missing and/or corrupted. +trait StorageLoader: SmtStorage + Sized { + /// Creates a storage backend for the given domain. + fn create(data_dir: &Path, domain: &'static str) -> Result; + + /// Loads an account tree, either from persistent storage or by rebuilding from DB. + fn load_account_tree( + self, + db: &mut Db, + ) -> impl std::future::Future, StateInitializationError>> + Send; + + /// Loads a nullifier tree, either from persistent storage or by rebuilding from DB. + fn load_nullifier_tree( + self, + db: &mut Db, + ) -> impl std::future::Future< + Output = Result>, StateInitializationError>, + > + Send; +} + +#[cfg(not(feature = "rocksdb"))] +impl StorageLoader for MemoryStorage { + fn create(_data_dir: &Path, _domain: &'static str) -> Result { + Ok(MemoryStorage::default()) + } + + async fn load_account_tree( + self, + db: &mut Db, + ) -> Result, StateInitializationError> { + let account_data = db.select_all_account_commitments().await?; + let smt_entries = account_data + .into_iter() + .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); + LargeSmt::with_entries(self, smt_entries) + .map_err(account_tree_large_smt_error_to_init_error) + } + + async fn load_nullifier_tree( + self, + db: &mut Db, + ) -> Result>, StateInitializationError> { + let nullifiers = db.select_all_nullifiers().await?; + let entries = nullifiers.into_iter().map(|info| (info.nullifier, info.block_num)); + NullifierTree::with_storage_from_entries(self, entries) + .map_err(StateInitializationError::FailedToCreateNullifierTree) + } +} + +#[cfg(feature = "rocksdb")] +impl StorageLoader for RocksDbStorage { + fn create(data_dir: &Path, domain: &'static str) -> Result { + let storage_path = data_dir.join(domain); + fs_err::create_dir_all(&storage_path) + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; + RocksDbStorage::open(RocksDbConfig::new(storage_path)) + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string())) + } + + async fn load_account_tree( + self, + db: &mut Db, + ) -> Result, StateInitializationError> { + // If RocksDB storage has data, load from it directly + let has_data = self + .has_leaves() + .map_err(|e| StateInitializationError::AccountTreeIoError(e.to_string()))?; + if has_data { + return load_smt(self); + } + + info!(target: COMPONENT, "RocksDB account tree storage is empty, populating from SQLite"); + let account_data = db.select_all_account_commitments().await?; + let smt_entries = account_data + .into_iter() + .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); + LargeSmt::with_entries(self, smt_entries) + .map_err(account_tree_large_smt_error_to_init_error) + } + + async fn load_nullifier_tree( + self, + db: &mut Db, + ) -> Result>, StateInitializationError> { + // If RocksDB storage has data, load from it directly + let has_data = self + .has_leaves() + .map_err(|e| StateInitializationError::NullifierTreeIoError(e.to_string()))?; + if has_data { + let smt = load_smt(self)?; + return Ok(NullifierTree::new_unchecked(smt)); + } + + info!(target: COMPONENT, "RocksDB nullifier tree storage is empty, populating from SQLite"); + let nullifiers = db.select_all_nullifiers().await?; + let entries = nullifiers.into_iter().map(|info| (info.nullifier, info.block_num)); + NullifierTree::with_storage_from_entries(self, entries) + .map_err(StateInitializationError::FailedToCreateNullifierTree) + } +} + /// Container for state that needs to be updated atomically. -struct InnerState +struct InnerState where S: SmtStorage, { @@ -92,10 +222,7 @@ where account_tree: AccountTreeWithHistory, } -impl InnerState -where - S: SmtStorage, -{ +impl InnerState { /// Returns the latest block number. fn latest_block_num(&self) -> BlockNumber { self.blockchain @@ -119,7 +246,7 @@ pub struct State { /// Read-write lock used to prevent writing to a structure while it is being used. /// /// The lock is writer-preferring, meaning the writer won't be starved. - inner: RwLock, + inner: RwLock>, /// Forest-related state `(SmtForest, storage_map_roots, vault_roots)` with its own lock. forest: RwLock, @@ -133,7 +260,7 @@ impl State { // CONSTRUCTOR // -------------------------------------------------------------------------------------------- - /// Loads the state from the `db`. + /// Loads the state from the data directory. #[instrument(target = COMPONENT, skip_all)] pub async fn load(data_path: &Path) -> Result { let data_directory = DataDirectory::load(data_path.to_path_buf()) @@ -151,8 +278,16 @@ impl State { let blockchain = load_mmr(&mut db).await?; let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); - let account_tree = load_account_tree(&mut db, latest_block_num).await?; - let nullifier_tree = load_nullifier_tree(&mut db).await?; + + let account_storage = TreeStorage::create(data_path, "accounttree")?; + let smt = account_storage.load_account_tree(&mut db).await?; + let account_tree = + AccountTree::new(smt).map_err(StateInitializationError::FailedToCreateAccountsTree)?; + let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); + + let nullifier_storage = TreeStorage::create(data_path, "nullifiertree")?; + let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; + let forest = load_smt_forest(&mut db, latest_block_num).await?; let inner = RwLock::new(InnerState { nullifier_tree, blockchain, account_tree }); @@ -1208,49 +1343,6 @@ async fn load_mmr(db: &mut Db) -> Result { Ok(chain_mmr) } -#[instrument(level = "info", target = COMPONENT, skip_all)] -async fn load_nullifier_tree( - db: &mut Db, -) -> Result>, StateInitializationError> { - let nullifiers = db.select_all_nullifiers().await?; - - // Convert nullifier data to entries for NullifierTree - // The nullifier value format is: block_num - let entries = nullifiers.into_iter().map(|info| (info.nullifier, info.block_num)); - - NullifierTree::with_storage_from_entries(MemoryStorage::default(), entries) - .map_err(StateInitializationError::FailedToCreateNullifierTree) -} - -#[instrument(level = "info", target = COMPONENT, skip_all)] -async fn load_account_tree( - db: &mut Db, - block_number: BlockNumber, -) -> Result, StateInitializationError> { - let account_data = Vec::from_iter(db.select_all_account_commitments().await?); - - let smt_entries = account_data - .into_iter() - .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); - - let smt = - LargeSmt::with_entries(MemoryStorage::default(), smt_entries).map_err(|e| match e { - LargeSmtError::Merkle(merkle_error) => { - StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) - }, - LargeSmtError::Storage(err) => { - // large_smt::StorageError is not `Sync` and hence `context` cannot be called - // which we want to and do - StateInitializationError::AccountTreeIoError(err.as_report()) - }, - })?; - - let account_tree = - AccountTree::new(smt).map_err(StateInitializationError::FailedToCreateAccountsTree)?; - - Ok(AccountTreeWithHistory::new(account_tree, block_number)) -} - /// Loads SMT forest with storage map and vault Merkle paths for all public accounts. #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num))] async fn load_smt_forest(