Skip to content

Run Benchmarks on EC2 Runner #143

Run Benchmarks on EC2 Runner

Run Benchmarks on EC2 Runner #143

Workflow file for this run

name: Run Benchmarks on EC2 Runner
on:
workflow_dispatch:
inputs:
runner_name:
required: true
type: string
instance_type:
required: true
type: string
enable_gpu:
required: true
type: string
provers:
required: false
type: string
default: 'sp1'
programs:
required: false
type: string
default: 'loop,fibonacci,tendermint,reth1,reth2'
filename:
required: false
type: string
default: 'benchmark'
trials:
required: false
type: string
default: '1'
sp1_ref:
required: false
type: string
additional_params:
required: false
type: string
default: '{"hashfns":"poseidon","shard_sizes":"22"}'
jobs:
run-benchmark:
name: Run Benchmark on ${{ inputs.runner_name }}
runs-on: ${{ inputs.runner_name }}
steps:
- name: Echo Workflow Inputs
run: |
echo "Runner Name: ${{ inputs.runner_name }}"
echo "Instance Type: ${{ inputs.instance_type }}"
echo "Enable GPU: ${{ inputs.enable_gpu }}"
echo "Provers: ${{ inputs.provers }}"
echo "Programs: ${{ inputs.programs }}"
echo "Filename: ${{ inputs.filename }}"
echo "Trials: ${{ inputs.trials }}"
echo "SP1 Ref: ${{ inputs.sp1_ref }}"
echo "Additional Params: ${{ inputs.additional_params }}"
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up Docker
uses: docker/setup-buildx-action@v3
- name: Parse additional parameters
id: parse-params
run: |
echo 'ADDITIONAL_PARAMS<<EOF' >> $GITHUB_ENV
echo '${{ inputs.additional_params }}' >> $GITHUB_ENV
echo 'EOF' >> $GITHUB_ENV
# - name: Cache Docker layers
# uses: actions/cache@v3
# with:
# path: /tmp/.buildx-cache
# key: ${{ runner.os }}-buildx-${{ github.sha }}
# restore-keys: |
# ${{ runner.os }}-buildx-
# - name: Build and cache Docker image
# uses: docker/build-push-action@v5
# with:
# context: .
# file: ./Dockerfile.gpu
# push: false
# load: true
# tags: zkvm-perf
# build-args: |
# SP1_REF=${{ inputs.sp1_ref }}
# platforms: linux/amd64
# cache-from: type=local,src=/tmp/.buildx-cache
# cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
# # This step is necessary to prevent the cache from growing indefinitely
# - name: Move cache
# run: |
# rm -rf /tmp/.buildx-cache
# mv /tmp/.buildx-cache-new /tmp/.buildx-cache
- name: Build Docker image
run: |
docker build -t zkvm-perf --platform linux/amd64 -f Dockerfile.gpu --build-arg SP1_REF=${{ inputs.sp1_ref }} .
- name: Make benchmarks directory
run: mkdir -p ${{ github.workspace }}/benchmarks
- name: rust-cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
~/.rustup/
key: rust-1.79.0-${{ hashFiles('**/Cargo.toml') }}
restore-keys: rust-1.79.0-
- name: Run benchmark
run: |
docker run ${{ inputs.enable_gpu == 'true' && '--gpus all' || '' }} --platform linux/amd64 \
-v /var/run/docker.sock:/var/run/docker.sock \
-v ${{ github.workspace }}/benchmarks:/usr/src/app/benchmarks \
-v ~/.cargo/registry:/root/.cargo/registry \
-v ~/.cargo/git:/root/.cargo/git \
-e RUST_BACKTRACE=full \
-e INSTANCE_TYPE=${{ inputs.instance_type }} \
--network host \
zkvm-perf \
"python3 sweep.py --filename ${{ inputs.filename }} --trials ${{ inputs.trials }} --programs $(echo ${{ inputs.programs }} | sed 's/,/ /g') --provers ${{ inputs.provers }} --hashfns ${{ fromJson(env.ADDITIONAL_PARAMS).hashfns }} --shard-sizes ${{ fromJson(env.ADDITIONAL_PARAMS).shard_sizes }}"
- name: List benchmark results
run: ls -la ${{ github.workspace }}/benchmarks
- name: Add instance type and SP1 ref to CSV
run: |
for file in ${{ github.workspace }}/benchmarks/*.csv; do
sed -i '1s/^/instance_type,sp1_ref,/' "$file"
sed -i "2,\$s|^|${{ inputs.instance_type }},${{ inputs.sp1_ref }},|" "$file"
done
- name: Upload benchmark results
uses: actions/upload-artifact@v3
with:
name: benchmark-results-${{ inputs.instance_type }}-${{ github.run_id }}
path: ${{ github.workspace }}/benchmarks/*.csv
- name: Print Results
run: |
cat ${{ github.workspace }}/benchmarks/*.csv