[DRAFT] Update benchmarks

.github/workflows/analysis.yml

@@ -133,13 +133,16 @@ jobs:
       - name: Set Rust toolchain override
         run: rustup override set ${{ steps.toolchain.outputs.name }}
 
+      - name: Install valgrind
+        run: sudo apt-get update && sudo apt-get install -y valgrind
+
       - name: Install cargo-udeps
         run: cargo install cargo-udeps
 
       - name: Run cargo udeps
-        # we only use openssl when the openssl-benchmarks feature is enabled.
-        # openssl is a dev-dependency so it can't be optional.
-        run: cargo udeps --workspace --all-targets --features openssl-benchmarks
+        # ring and openssl are optional dependencies in aws-lc-rs-testing,
+        # gated by ring-benchmarks and openssl-benchmarks features respectively.
+        run: cargo udeps --workspace --all-targets --features ring-benchmarks,openssl-benchmarks
         env:
           RUSTC_WRAPPER: ""
 
@@ -404,7 +407,7 @@ jobs:
       - name: Install readelf
         run: sudo apt-get update && sudo apt-get install -y binutils
       - name: Build project
-        run: cargo build --workspace --all-targets --release
+        run: cargo build -p aws-lc-rs -p aws-lc-sys -p aws-lc-fips-sys --all-targets --release
       - name: Check for GNU-stack section in object files
         run: |
           echo "Checking for .note.GNU-stack section in all object files..."

.github/workflows/benchmarks.yml

@@ -0,0 +1,266 @@
+name: Benchmarks
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUST_BACKTRACE: 1
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  benchmark:
+    name: Benchmark (${{ matrix.target }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-22.04
+            target: x86_64-unknown-linux-gnu
+          - os: ubuntu-22.04-arm
+            target: aarch64-unknown-linux-gnu
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install valgrind
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-${{ matrix.target }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.target }}-cargo-bench-
+
+      - name: Build benchmark tool (candidate)
+        run: cargo build --release -p aws-lc-rs-bench
+
+      - name: Checkout baseline (main branch)
+        if: github.event_name == 'pull_request'
+        run: |
+          git worktree add ../baseline origin/${{ github.base_ref }}
+          cd ../baseline
+          git submodule update --init --recursive
+
+      - name: Build benchmark tool (baseline)
+        if: github.event_name == 'pull_request'
+        run: |
+          cd ../baseline
+          cargo build --release -p aws-lc-rs-bench
+
+      - name: Run baseline benchmarks
+        if: github.event_name == 'pull_request'
+        run: |
+          cd ../baseline
+          ./target/release/aws-lc-rs-bench run-all --output-dir baseline-results
+
+      - name: Run candidate benchmarks
+        run: |
+          ./target/release/aws-lc-rs-bench run-all --output-dir candidate-results
+
+      - name: Compare results
+        if: github.event_name == 'pull_request'
+        id: compare
+        run: |
+          ./target/release/aws-lc-rs-bench compare ../baseline/baseline-results candidate-results > benchmark-report-${{ matrix.target }}.md
+          cat benchmark-report-${{ matrix.target }}.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results-${{ matrix.target }}
+          path: |
+            candidate-results/
+            benchmark-report-${{ matrix.target }}.md
+          retention-days: 30
+
+      - name: Store main branch results
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@v4
+        with:
+          name: main-benchmark-results-${{ matrix.target }}
+          path: candidate-results/
+          retention-days: 90
+
+  benchmark-fips:
+    name: Benchmark FIPS (${{ matrix.target }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-22.04
+            target: x86_64-unknown-linux-gnu
+          - os: ubuntu-22.04-arm
+            target: aarch64-unknown-linux-gnu
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install valgrind
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind
+
+      - name: Install Go (required for FIPS build)
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-${{ matrix.target }}-cargo-bench-fips-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.target }}-cargo-bench-fips-
+
+      - name: Build benchmark tool with FIPS (candidate)
+        run: cargo build --release -p aws-lc-rs-bench --features fips
+
+      - name: Checkout baseline (main branch)
+        if: github.event_name == 'pull_request'
+        run: |
+          git worktree add ../baseline origin/${{ github.base_ref }}
+          cd ../baseline
+          git submodule update --init --recursive
+
+      - name: Build benchmark tool with FIPS (baseline)
+        if: github.event_name == 'pull_request'
+        run: |
+          cd ../baseline
+          cargo build --release -p aws-lc-rs-bench --features fips
+
+      - name: Run baseline FIPS benchmarks
+        if: github.event_name == 'pull_request'
+        run: |
+          cd ../baseline
+          ./target/release/aws-lc-rs-bench run-all --output-dir baseline-fips-results
+
+      - name: Run candidate FIPS benchmarks
+        run: |
+          ./target/release/aws-lc-rs-bench run-all --output-dir candidate-fips-results
+
+      - name: Compare FIPS results
+        if: github.event_name == 'pull_request'
+        id: compare
+        run: |
+          ./target/release/aws-lc-rs-bench compare ../baseline/baseline-fips-results candidate-fips-results > benchmark-fips-report-${{ matrix.target }}.md
+          cat benchmark-fips-report-${{ matrix.target }}.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload FIPS benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-fips-results-${{ matrix.target }}
+          path: |
+            candidate-fips-results/
+            benchmark-fips-report-${{ matrix.target }}.md
+          retention-days: 30
+
+      - name: Store main branch FIPS results
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@v4
+        with:
+          name: main-benchmark-fips-results-${{ matrix.target }}
+          path: candidate-fips-results/
+          retention-days: 90
+
+  post-comment:
+    name: Post PR Comment
+    needs: [benchmark, benchmark-fips]
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all benchmark reports
+        uses: actions/download-artifact@v4
+        with:
+          pattern: benchmark-*-results-*
+          merge-multiple: true
+
+      - name: Combine reports
+        run: |
+          echo "<!-- aws-lc-rs-benchmark-results -->" > comment-body.md
+          echo "" >> comment-body.md
+          echo "# Benchmark Results" >> comment-body.md
+          echo "" >> comment-body.md
+          echo "## Standard Build" >> comment-body.md
+          echo "" >> comment-body.md
+          for f in benchmark-report-*.md; do
+            if [ -f "$f" ]; then
+              target=$(echo "$f" | sed 's/benchmark-report-\(.*\)\.md/\1/')
+              echo "### Target: \`$target\`" >> comment-body.md
+              echo "" >> comment-body.md
+              cat "$f" >> comment-body.md
+              echo "" >> comment-body.md
+            fi
+          done
+          echo "## FIPS Build" >> comment-body.md
+          echo "" >> comment-body.md
+          for f in benchmark-fips-report-*.md; do
+            if [ -f "$f" ]; then
+              target=$(echo "$f" | sed 's/benchmark-fips-report-\(.*\)\.md/\1/')
+              echo "### Target: \`$target\`" >> comment-body.md
+              echo "" >> comment-body.md
+              cat "$f" >> comment-body.md
+              echo "" >> comment-body.md
+            fi
+          done
+          echo "" >> comment-body.md
+          echo "---" >> comment-body.md
+          echo "> **Note**: Benchmark results are measured using CPU instruction counts via Valgrind's callgrind tool." >> comment-body.md
+          echo "> Changes greater than 2% are considered significant." >> comment-body.md
+          echo ">" >> comment-body.md
+          echo "> ✅ = improvement, ⚠️ = regression" >> comment-body.md
+
+      - name: Find existing comment
+        uses: peter-evans/find-comment@v3
+        id: find-comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: '<!-- aws-lc-rs-benchmark-results -->'
+
+      - name: Create or update comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.find-comment.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: comment-body.md
+          edit-mode: replace

.github/workflows/tests.yml

@@ -55,7 +55,7 @@ jobs:
         run: cargo test ${{ matrix.args }}
       - name: Run extra tests
         working-directory: ./aws-lc-rs-testing
-        run: cargo test --all-targets
+        run: cargo test --all-targets --features ring-benchmarks,openssl-benchmarks
 
   bindgen-test:
     if: github.repository_owner == 'aws'

.gitignore

@@ -7,6 +7,7 @@
 Cargo.lock
 deps/aws-lc-sys/src/bindings.rs
 /flamegraph.svg
+/results/
 
 # These are backup files generated by rustfmt
 **/*.rs.bk

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "aws-lc-sys",
     "aws-lc-fips-sys",
     "aws-lc-rs-testing",
+    "aws-lc-rs-bench",
     "links-testing",
 ]
 exclude = [
@@ -28,13 +29,16 @@ zeroize = "1.8.1"
 # The dependencies below do not affect our MSRV. They are only used as
 # a dev-dependency or only used outside of our public crates.
 # These can be updated to the latest versions.
+anyhow = "1.0"
 clap = "4.4"
 criterion = "0.8.2"
 hex = "0.4.3"
 lazy_static = "1.5.0"
 openssl = "0.10.73"
 paste = "1.0.15"
 ring = "0.17.14"
+serde = "1.0"
+serde_json = "1.0"
 toml_edit = "0.25.0"
 
 [profile.bench]

aws-lc-fips-sys/builder/main.rs

@@ -696,6 +696,7 @@ const PRELUDE: &str = r"
     clippy::cast_possible_truncation,
     clippy::cast_possible_wrap,
     clippy::default_trait_access,
+    clippy::doc_markdown,
     clippy::missing_safety_doc,
     clippy::must_use_candidate,
     clippy::not_unsafe_ptr_arg_deref,

aws-lc-fips-sys/src/lib.rs

@@ -60,6 +60,7 @@ platform_binding!(
     clippy::cast_possible_truncation,
     clippy::cast_possible_wrap,
     clippy::default_trait_access,
+    clippy::doc_markdown,
     clippy::missing_safety_doc,
     clippy::must_use_candidate,
     clippy::not_unsafe_ptr_arg_deref,

aws-lc-rs-bench/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "aws-lc-rs-bench"
+version = "0.1.0"
+edition = "2021"
+rust-version = "1.75.0"
+publish = false
+license = "Apache-2.0 OR ISC"
+description = "CI benchmarking tool for aws-lc-rs"
+
+[features]
+default = []
+fips = ["aws-lc-rs/fips"]
+
+[dependencies]
+aws-lc-rs = { path = "../aws-lc-rs", default-features = true }
+clap = { workspace = true, features = ["derive"] }
+anyhow = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+# For instruction counting with valgrind/callgrind
+crabgrind = "0.1"