Merge pull request #1158 from GitGab19/fix-benches-false-alarm-on-ci

Fix bencher false alarms on CI
stratum-mining · Sep 9, 2024 · 8da5e5f · 8da5e5f
2 parents 708e5e4 + ed7dbdc
commit 8da5e5f
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 22 deletions.
diff --git a/.github/workflows/run-benchmarks.yaml b/.github/workflows/run-benchmarks.yaml
@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - dev
 
 jobs:
   benchmark_sv1_criterion:

diff --git a/.github/workflows/track-benchmarks.yaml b/.github/workflows/track-benchmarks.yaml
@@ -56,15 +56,16 @@ jobs:
             let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'}));
             core.exportVariable("PR_HEAD", `${prEvent.number}/merge`);
             core.exportVariable("PR_BASE", prEvent.pull_request.base.ref);
-            core.exportVariable("PR_DEFAULT", prEvent.pull_request.base.repo.default_branch);
+            core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha);
             core.exportVariable("PR_NUMBER", prEvent.number);
       - uses: bencherdev/bencher@main
       - name: Track Benchmarks with Bencher
         run: |
           bencher run \
-          --if-branch '${{ env.PR_HEAD }}' \
-          --else-if-branch '${{ env.PR_BASE }}' \
-          --else-if-branch '${{ env.PR_DEFAULT }}' \
+          --branch '${{ env.PR_HEAD }}' \
+          --branch-start-point '${{ env.PR_BASE }}' \
+          --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \
+          --branch-reset \
           --ci-number '${{ env.PR_NUMBER }}' \
           --github-actions "${{ secrets.GITHUB_TOKEN }}" \
           --token "${{ secrets.BENCHER_API_TOKEN }}" \
@@ -118,17 +119,18 @@ jobs:
           script: |
             let fs = require('fs');
             let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'}));
-            core.exportVariable("PR_HEAD", prEvent.pull_request.head.ref);
+            core.exportVariable("PR_HEAD", `${prEvent.number}/merge`);
             core.exportVariable("PR_BASE", prEvent.pull_request.base.ref);
-            core.exportVariable("PR_DEFAULT", prEvent.pull_request.base.repo.default_branch);
+            core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha);
             core.exportVariable("PR_NUMBER", prEvent.number);
       - uses: bencherdev/bencher@main
       - name: Track Benchmarks with Bencher
         run: |
           bencher run \
-          --if-branch '${{ env.PR_HEAD }}' \
-          --else-if-branch '${{ env.PR_BASE }}' \
-          --else-if-branch '${{ env.PR_DEFAULT }}' \
+          --branch '${{ env.PR_HEAD }}' \
+          --branch-start-point '${{ env.PR_BASE }}' \
+          --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \
+          --branch-reset \
           --ci-number '${{ env.PR_NUMBER }}' \
           --github-actions "${{ secrets.GITHUB_TOKEN }}" \
           --token "${{ secrets.BENCHER_API_TOKEN }}" \
@@ -182,17 +184,18 @@ jobs:
           script: |
             let fs = require('fs');
             let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'}));
-            core.exportVariable("PR_HEAD", prEvent.pull_request.head.ref);
+            core.exportVariable("PR_HEAD", `${prEvent.number}/merge`);
             core.exportVariable("PR_BASE", prEvent.pull_request.base.ref);
-            core.exportVariable("PR_DEFAULT", prEvent.pull_request.base.repo.default_branch);
+            core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha);
             core.exportVariable("PR_NUMBER", prEvent.number);
       - uses: bencherdev/bencher@main
       - name: Track Benchmarks with Bencher
         run: |
           bencher run \
-          --if-branch '${{ env.PR_HEAD }}' \
-          --else-if-branch '${{ env.PR_BASE }}' \
-          --else-if-branch '${{ env.PR_DEFAULT }}' \
+          --branch '${{ env.PR_HEAD }}' \
+          --branch-start-point '${{ env.PR_BASE }}' \
+          --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \
+          --branch-reset \
           --ci-number '${{ env.PR_NUMBER }}' \
           --github-actions "${{ secrets.GITHUB_TOKEN }}" \
           --token "${{ secrets.BENCHER_API_TOKEN }}" \
@@ -246,17 +249,18 @@ jobs:
           script: |
             let fs = require('fs');
             let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'}));
-            core.exportVariable("PR_HEAD", prEvent.pull_request.head.ref);
+            core.exportVariable("PR_HEAD", `${prEvent.number}/merge`);
             core.exportVariable("PR_BASE", prEvent.pull_request.base.ref);
-            core.exportVariable("PR_DEFAULT", prEvent.pull_request.base.repo.default_branch);
+            core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha);
             core.exportVariable("PR_NUMBER", prEvent.number);
       - uses: bencherdev/bencher@main
       - name: Track Benchmarks with Bencher
         run: |
           bencher run \
-          --if-branch '${{ env.PR_HEAD }}' \
-          --else-if-branch '${{ env.PR_BASE }}' \
-          --else-if-branch '${{ env.PR_DEFAULT }}' \
+          --branch '${{ env.PR_HEAD }}' \
+          --branch-start-point '${{ env.PR_BASE }}' \
+          --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \
+          --branch-reset \
           --ci-number '${{ env.PR_NUMBER }}' \
           --github-actions "${{ secrets.GITHUB_TOKEN }}" \
           --token "${{ secrets.BENCHER_API_TOKEN }}" \

diff --git a/benches/benches/src/sv1/criterion_sv1_benchmark.rs b/benches/benches/src/sv1/criterion_sv1_benchmark.rs
@@ -220,7 +220,7 @@ fn benchmark_submit_serialize_deserialize_handle(c: &mut Criterion, mut client:
 
 fn main() {
     let mut criterion = Criterion::default()
-        .sample_size(50)
+        .sample_size(100)
         .measurement_time(std::time::Duration::from_secs(5));
     let client = Client::new(90);
     benchmark_get_subscribe(&mut criterion, client.clone());

diff --git a/benches/benches/src/sv2/criterion_sv2_benchmark.rs b/benches/benches/src/sv2/criterion_sv2_benchmark.rs
@@ -197,7 +197,7 @@ fn client_sv2_handle_message_common(c: &mut Criterion) {
 
 fn main() {
     let mut criterion = Criterion::default()
-        .sample_size(50)
+        .sample_size(100)
         .measurement_time(std::time::Duration::from_secs(5));
     client_sv2_setup_connection(&mut criterion);
     client_sv2_setup_connection_serialize(&mut criterion);