diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 016aeba77..9c2168543 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,12 @@ env: RUST_CACHE_KEY: rust-cache-2026.02.02 # Reduce cache usage by removing debug information. CARGO_PROFILE_DEV_DEBUG: 0 + ROCKSDB_STATIC: true + ROCKSDB_LIB_DIR: ${{ github.workspace }}/rocksdb-artifact + ROCKSDB_COMPILE: false + LZ4_LIB_DIR: ${{ github.workspace }}/rocksdb-artifact + LZ4_STATIC: true + LIBZ_SYS_STATIC: 1 # Limits workflow concurrency to only the latest commit in the PR. concurrency: @@ -36,6 +42,12 @@ concurrency: cancel-in-progress: true jobs: + rocksdb-static: + uses: ./.github/workflows/rocksdb-static.yml + with: + runs-on: ubuntu-24.04 + checkout-ref: ${{ github.ref }} + # =============================================================================================== # Conventional builds, lints and tests that re-use a single cache for efficiency # =============================================================================================== @@ -43,12 +55,20 @@ jobs: # Normal cargo build that populates a cache for all subsequent jobs to re-use. build: runs-on: ubuntu-24.04 + needs: [rocksdb-static] steps: - uses: actions/checkout@v6 - name: Cleanup large tools for build space uses: ./.github/actions/cleanup-runner - name: Install RocksDB uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 @@ -65,6 +85,15 @@ jobs: needs: [build] steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 @@ -81,6 +110,15 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: taiki-e/install-action@v2 @@ -105,6 +143,15 @@ jobs: - uses: actions/checkout@v6 - name: Cleanup large tools for build space uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 @@ -124,6 +171,15 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - name: Install protobuf @@ -148,6 +204,15 @@ jobs: DATA_DIR: /tmp/store steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - uses: Swatinem/rust-cache@v2 @@ -194,8 +259,18 @@ jobs: client-wasm: name: wasm targets runs-on: ubuntu-24.04 + needs: [rocksdb-static] steps: - uses: actions/checkout@v6 + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup update --no-self-update - name: cargo build diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a5d6e3cae..806271bcb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,12 +11,27 @@ on: permissions: contents: read +env: + ROCKSDB_STATIC: true + ROCKSDB_LIB_DIR: ${{ github.workspace }}/rocksdb-artifact + ROCKSDB_COMPILE: false + LZ4_LIB_DIR: ${{ github.workspace }}/rocksdb-artifact + LZ4_STATIC: true + LIBZ_SYS_STATIC: 1 + jobs: + rocksdb-static: + uses: ./.github/workflows/rocksdb-static.yml + with: + runs-on: ubuntu-24.04 + checkout-ref: refs/heads/next + # Run tests on the beta channel to provide feedback for Rust team. beta-test: name: test on beta channel runs-on: ubuntu-24.04 timeout-minutes: 30 + needs: rocksdb-static steps: - uses: actions/checkout@v6 with: @@ -25,6 +40,13 @@ jobs: uses: ./.github/actions/cleanup-runner - name: Install RocksDB uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Rustup run: rustup install beta && rustup default beta - uses: taiki-e/install-action@v2 @@ -39,6 +61,7 @@ jobs: check-features: name: feature combinations runs-on: ubuntu-24.04 + needs: rocksdb-static steps: - uses: actions/checkout@v6 with: @@ -47,6 +70,13 @@ jobs: uses: ./.github/actions/cleanup-runner - name: Install RocksDB uses: ./.github/actions/install-rocksdb + - name: Download RocksDB static library + uses: actions/download-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + - name: Verify RocksDB static library + run: bash .github/verify-rocksdb-static.sh - name: Install rust run: rustup update --no-self-update - name: Install cargo-hack @@ -58,6 +88,7 @@ jobs: msrv: name: msrv check runs-on: ubuntu-24.04 + needs: rocksdb-static steps: - uses: actions/checkout@v6 with: diff --git a/.github/workflows/publish-debian-all.yml b/.github/workflows/publish-debian-all.yml index 76e65d0eb..e45ddc65b 100644 --- a/.github/workflows/publish-debian-all.yml +++ b/.github/workflows/publish-debian-all.yml @@ -19,6 +19,11 @@ permissions: contents: write jobs: + rocksdb-static: + uses: ./.github/workflows/rocksdb-static.yml + with: + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + checkout-ref: ${{ github.ref }} publish-node: name: Publish Node ${{ matrix.arch }} Debian strategy: diff --git a/.github/workflows/publish-debian.yml b/.github/workflows/publish-debian.yml index d17d06532..108512168 100644 --- a/.github/workflows/publish-debian.yml +++ b/.github/workflows/publish-debian.yml @@ -47,6 +47,11 @@ permissions: contents: write jobs: + rocksdb-static: + uses: ./.github/workflows/rocksdb-static.yml + with: + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + checkout-ref: ${{ github.ref }} publish: name: Publish ${{ inputs.package }} ${{ matrix.arch }} Debian strategy: diff --git a/.github/workflows/rocksdb-static.yml b/.github/workflows/rocksdb-static.yml new file mode 100644 index 000000000..5bb30c39d --- /dev/null +++ b/.github/workflows/rocksdb-static.yml @@ -0,0 +1,74 @@ +name: rocksdb-static + +on: + workflow_call: + inputs: + runs-on: + required: true + type: string + checkout-ref: + required: false + type: string + +permissions: + contents: read + +env: + # Reduce cache usage by removing debug information. + CARGO_PROFILE_DEV_DEBUG: 0 + +jobs: + rocksdb-static: + name: rocksdb-static + runs-on: ${{ inputs.runs-on }} + timeout-minutes: 30 + env: + ROCKSDB_RUST_VERSION: "" + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.checkout-ref || github.ref }} + - name: Cleanup large tools for build space + uses: ./.github/actions/cleanup-runner + - name: Install RocksDB + uses: ./.github/actions/install-rocksdb + - name: Rustup + run: rustup update --no-self-update + - name: Resolve RocksDB version + run: | + set -euo pipefail + rocksdb_version=$(cargo tree --locked --depth=0 -i librocksdb-sys | tail -n 1 | awk '{print $2}' | sed 's/^v//') + if [ -z "$rocksdb_version" ]; then + echo "::error::Unable to determine librocksdb-sys version from cargo tree." + exit 1 + fi + echo "ROCKSDB_RUST_VERSION=$rocksdb_version" >> $GITHUB_ENV + - name: RocksDB cache date + id: rocksdb-date + run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT + - name: Cache RocksDB static library + id: rocksdb-cache + uses: actions/cache@v4 + with: + path: rocksdb-artifact + key: rocksdb-${{ runner.os }}-${{ runner.arch }}-${{ env.ROCKSDB_RUST_VERSION }}-${{ steps.rocksdb-date.outputs.date }}-${{ hashFiles('Cargo.lock') }} + - name: Build rocksdb.a + if: steps.rocksdb-cache.outputs.cache-hit != 'true' + run: | + set -euo pipefail + CARGO_TARGET_DIR=rocksdb-target cargo build -p librocksdb-sys@${{ env.ROCKSDB_RUST_VERSION }} --locked + mkdir -p rocksdb-artifact + for lib in librocksdb.a liblz4.a; do + lib_path="$(find rocksdb-target -name "$lib" -print -quit)" + if [ -z "$lib_path" ]; then + echo "::error::Missing $lib." + exit 1 + fi + cp "$lib_path" "rocksdb-artifact/$lib" + done + - name: Upload RocksDB static library + uses: actions/upload-artifact@v4 + with: + name: rocksdb-static-${{ runner.arch }} + path: rocksdb-artifact + if-no-files-found: error diff --git a/Cargo.lock b/Cargo.lock index 65360401b..d41133072 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2850,6 +2850,7 @@ dependencies = [ "http", "miden-node-grpc-error-macro", "miden-node-proto-build", + "miden-node-rocksdb-cxx-linkage-fix", "miden-node-utils", "miden-protocol", "miden-standards", diff --git a/crates/proto/Cargo.toml b/crates/proto/Cargo.toml index 6d3589ca3..2e9767f88 100644 --- a/crates/proto/Cargo.toml +++ b/crates/proto/Cargo.toml @@ -33,7 +33,8 @@ assert_matches = { workspace = true } proptest = { version = "1.7" } [build-dependencies] -fs-err = { workspace = true } -miden-node-proto-build = { features = ["internal"], workspace = true } -miette = { version = "7.6" } -tonic-prost-build = { workspace = true } +fs-err = { workspace = true } +miden-node-proto-build = { features = ["internal"], workspace = true } +miden-node-rocksdb-cxx-linkage-fix = { workspace = true } +miette = { version = "7.6" } +tonic-prost-build = { workspace = true } diff --git a/crates/proto/build.rs b/crates/proto/build.rs index b0ac773a7..5a39c1d54 100644 --- a/crates/proto/build.rs +++ b/crates/proto/build.rs @@ -22,6 +22,8 @@ fn main() -> miette::Result<()> { println!("cargo::rerun-if-changed=../../proto/proto"); println!("cargo::rerun-if-env-changed=BUILD_PROTO"); + miden_node_rocksdb_cxx_linkage_fix::configure(); + // Skip this build script in BUILD_PROTO environment variable is not set to `1`. if env::var("BUILD_PROTO").unwrap_or("0".to_string()) == "0" { return Ok(()); diff --git a/crates/rocksdb-cxx-linkage-fix/src/lib.rs b/crates/rocksdb-cxx-linkage-fix/src/lib.rs index 9eaae82fd..a20f550bd 100644 --- a/crates/rocksdb-cxx-linkage-fix/src/lib.rs +++ b/crates/rocksdb-cxx-linkage-fix/src/lib.rs @@ -17,7 +17,8 @@ pub fn configure() { fn should_link_cpp_stdlib() -> bool { let rocksdb_compile = env::var("ROCKSDB_COMPILE").unwrap_or_default(); - let rocksdb_compile_disabled = matches!(rocksdb_compile.as_str(), "0" | "false" | "FALSE"); + let rocksdb_compile_disabled = + rocksdb_compile.is_empty() || matches!(rocksdb_compile.as_str(), "0" | "false" | "FALSE"); let rocksdb_static = env::var("ROCKSDB_STATIC").is_ok(); let rocksdb_lib_dir_set = env::var("ROCKSDB_LIB_DIR").is_ok();