Pr benchmarking #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Benchmarks | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| env: | |
| CERBERUS_IMAGE_ID: ghcr.io/rems-project/cerberus/cn:release | |
| # cancel in-progress job when a new push is performed | |
| concurrency: | |
| group: ci-pr-bench-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark: | |
| name: Performance benchmarks | |
| strategy: | |
| matrix: | |
| # version: [4.12.0, 4.14.1] | |
| version: [4.14.1] | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 100 # this is to make sure we obtain the target base commit | |
| - name: System dependencies (ubuntu) | |
| run: | | |
| sudo apt install build-essential libgmp-dev z3 opam cmake | |
| - name: Restore cached opam | |
| id: cache-opam-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.opam | |
| key: ${{ matrix.version }} | |
| - name: Setup opam | |
| if: steps.cache-opam-restore.outputs.cache-hit != 'true' | |
| run: | | |
| opam init --yes --no-setup --shell=sh --compiler=${{ matrix.version }} | |
| opam install --deps-only --yes ./cerberus-lib.opam | |
| opam switch create with_coq ${{ matrix.version }} | |
| eval $(opam env --switch=with_coq) | |
| opam repo add --yes --this-switch coq-released https://coq.inria.fr/opam/released | |
| opam pin --yes -n coq-struct-tact https://github.com/uwplse/StructTact.git | |
| opam repo add --yes --this-switch iris-dev https://gitlab.mpi-sws.org/iris/opam.git | |
| opam pin --yes -n coq-sail-stdpp https://github.com/rems-project/coq-sail.git#f319aad | |
| opam pin --yes -n coq-cheri-capabilities https://github.com/rems-project/coq-cheri-capabilities.git | |
| opam install --deps-only --yes ./cerberus-lib.opam ./cerberus-cheri.opam | |
| - name: Save cached opam | |
| if: steps.cache-opam-restore.outputs.cache-hit != 'true' | |
| id: cache-opam-save | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ~/.opam | |
| key: ${{ steps.cache-opam-restore.outputs.cache-primary-key }} | |
| - name: Install Cerberus | |
| run: | | |
| opam switch ${{ matrix.version }} | |
| eval $(opam env --switch=${{ matrix.version }}) | |
| opam pin --yes --no-action add cerberus-lib . | |
| opam pin --yes --no-action add cerberus . | |
| opam install --yes cerberus | |
| - name: Download cvc5 release | |
| uses: robinraju/release-downloader@v1 | |
| with: | |
| repository: cvc5/cvc5 | |
| tag: cvc5-1.1.2 | |
| fileName: cvc5-Linux-static.zip | |
| - name: Unzip and install cvc5 | |
| run: | | |
| unzip cvc5-Linux-static.zip | |
| chmod +x cvc5-Linux-static/bin/cvc5 | |
| sudo cp cvc5-Linux-static/bin/cvc5 /usr/local/bin/ | |
| - name: Install CN | |
| run: | | |
| opam switch ${{ matrix.version }} | |
| eval $(opam env --switch=${{ matrix.version }}) | |
| opam pin --yes --no-action add cn . | |
| opam install --yes cn ocamlformat.0.26.2 | |
| - name: Run benchmark on PR | |
| run: | | |
| opam switch ${{ matrix.version }} | |
| eval $(opam env --switch=${{ matrix.version }}) | |
| cd tests; USE_OPAM='' ./run-ci-benchmarks.sh; mv benchmark-data.json benchmark-data-pr.json | |
| cd .. | |
| - name: Switch to target branch | |
| run: | | |
| git checkout ${{ github.event.pull_request.base.sha }} | |
| - name: Install Cerberus | |
| run: | | |
| opam switch ${{ matrix.version }} | |
| eval $(opam env --switch=${{ matrix.version }}) | |
| opam pin --yes --no-action add cerberus-lib . | |
| opam pin --yes --no-action add cerberus . | |
| opam install --yes cerberus | |
| - name: Install CN | |
| run: | | |
| opam switch ${{ matrix.version }} | |
| eval $(opam env --switch=${{ matrix.version }}) | |
| opam pin --yes --no-action add cn . | |
| opam install --yes cn ocamlformat.0.26.2 | |
| - name: Run benchmark on baseline | |
| run: | | |
| opam switch ${{ matrix.version }} | |
| eval $(opam env --switch=${{ matrix.version }}) | |
| cd tests; USE_OPAM='' ./run-ci-benchmarks.sh; mv benchmark-data.json benchmark-data-baseline.json | |
| cd .. | |
| - name: Compare results | |
| run: | | |
| echo 'Benchmark comparison for [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR) vs [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (baseline).' >> tests/benchmark-comparison.txt | |
| echo '```' >> tests/benchmark-comparison.txt | |
| echo tests/compare-benchmarks.py tests/benchmark-data-baseline.json tests/benchmark-data-pr.json >> tests/benchmark-comparison.txt | |
| echo '```' >> tests/benchmark-comparison.txt | |
| - name: 'Comment PR' | |
| uses: actions/[email protected] | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| github.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: require('fs').readFileSync('tests/benchmark-comparison.txt').toString() | |
| }); |