ci: benchmark: Use TypeScript code to manage competing benchmark

KSXGitHub · May 23, 2021 · 625ad71 · 625ad71
1 parent 55b7b10
commit 625ad71
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 64 deletions.
diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
@@ -138,66 +138,3 @@ jobs:
         with:
           name: benchmark-reports
           path: tmp.benchmark-report.*
-
-      - name: 'Benchmark: len'
-        run: |
-          commands=(
-            'pdu --quantity=len tmp.sample'
-            'dust --apparent-size tmp.sample'
-            'dutree tmp.sample'
-            'du --apparent-size tmp.sample'
-          )
-          hyperfine --warmup=3 "${commands[@]}"
-
-      - name: 'Benchmark: blksize'
-        run: |
-          commands=(
-            'pdu --quantity=blksize tmp.sample'
-            'dust tmp.sample'
-            'dutree --usage tmp.sample'
-            'du tmp.sample'
-          )
-          hyperfine --warmup=3 "${commands[@]}"
-
-      - name: 'Benchmark: top-down'
-        run: |
-          commands=(
-            'pdu --top-down tmp.sample'
-            'dust --apparent-size --reverse tmp.sample'
-            'dutree tmp.sample'
-          )
-          hyperfine --warmup=3 "${commands[@]}"
-
-      - name: 'Benchmark: summary'
-        run: |
-          commands=(
-            'pdu --max-depth=1 tmp.sample'
-            'dutree --summary tmp.sample'
-            'du --apparent-size --total tmp.sample'
-          )
-          hyperfine --warmup=3 "${commands[@]}"
-
-      - name: 'Benchmark: extreme details'
-        run: |
-          commands=(
-            'pdu --min-ratio=0 tmp.sample'
-            'dutree tmp.sample'
-            'du --apparent-size tmp.sample'
-          )
-          hyperfine --warmup=3 "${commands[@]}"
-
-      - name: 'Benchmark: no sort'
-        run: |
-          commands=(
-            'pdu --no-sort tmp.sample'
-            'du --apparent-size tmp.sample'
-          )
-          hyperfine --warmup=3 "${commands[@]}"
-
-      - name: 'Benchmark: no sort, summary'
-        run: |
-          commands=(
-            'pdu --no-sort --max-depth=1 tmp.sample'
-            'du --apparent-size --total tmp.sample'
-          )
-          hyperfine --warmup=3 "${commands[@]}"
diff --git a/ci/github-actions/benchmark.ts b/ci/github-actions/benchmark.ts
@@ -2,13 +2,25 @@ import console from 'console'
 import exec from 'exec-inline'
 import process from 'process'
 import shCmd from 'shell-escape'
-import { SELF_BENCHMARK_MATRIX, SELF_BENCHMARK_HYPERFINE_NAMES, parseSelfBenchmarkCategory } from './benchmark/matrix'
+import {
+  SELF_BENCHMARK_MATRIX,
+  SELF_BENCHMARK_HYPERFINE_NAMES,
+  COMPETING_BENCHMARK_MATRIX,
+  parseSelfBenchmarkCategory,
+} from './benchmark/matrix'
 import * as reportFiles from './benchmark/report-files'
 import STRICT_BASH from './benchmark/strict-bash'
 
 const pduTargets = process.argv.slice(2)
 const errexit = (param: { readonly status: number | null }) => param.status !== 0
 
+function section(title: string) {
+  console.error(title)
+  console.error('='.repeat(title.length))
+  console.error()
+}
+
+section('Compare benchmark of pdu against other versions of itself')
 for (const { category, units } of SELF_BENCHMARK_MATRIX) {
   const { commandSuffix, reportName } = parseSelfBenchmarkCategory(category)
   console.error({ category, commandSuffix, reportName })
@@ -20,3 +32,19 @@ for (const { category, units } of SELF_BENCHMARK_MATRIX) {
   exec(...STRICT_BASH, '-c', shellCommand).exit(errexit)
   console.error()
 }
+
+section('Compare benchmark of pdu against its competitors')
+for (const { id, pduCliArgs, competitors } of COMPETING_BENCHMARK_MATRIX) {
+  const commands = [
+    `pdu ${pduCliArgs.join(' ')} tmp.sample`,
+    ...competitors.map(argv => `${argv.join(' ')} tmp.sample` as const),
+  ] as const
+  console.error({ id, commands })
+  const reportName = `competing.${id}` as const
+  const exportReports = reportFiles.hyperfineArgs(reportName)
+  const commandLog = reportFiles.getFileName(reportName, 'log')
+  const hyperfineCommand = shCmd(['hyperfine', '--warmup=3', ...exportReports, ...commands])
+  const shellCommand = `${hyperfineCommand} 2>&1 | tee ${commandLog}`
+  exec(...STRICT_BASH, '-c', shellCommand).exit(errexit)
+  console.error()
+}
diff --git a/ci/github-actions/benchmark/matrix.ts b/ci/github-actions/benchmark/matrix.ts
@@ -74,3 +74,68 @@ export function getSelfBenchmarkHyperfineName<Version extends string>(
 }
 
 export const SELF_BENCHMARK_HYPERFINE_NAMES = UNITS.map(getSelfBenchmarkHyperfineName)
+
+export interface CompetingBenchmarkCategory {
+  readonly id: string
+  readonly pduCliArgs: readonly string[]
+  readonly competitors: ReadonlyArray<readonly [string, ...string[]]>
+}
+
+export const COMPETING_BENCHMARK_MATRIX: readonly CompetingBenchmarkCategory[] = [
+  {
+    id: 'len',
+    pduCliArgs: ['--quantity=len'],
+    competitors: [
+      ['dust', '--apparent-size'],
+      ['dutree'],
+      ['du', '--apparent-size'],
+    ],
+  },
+  {
+    id: 'blksize',
+    pduCliArgs: ['--quantity=blksize'],
+    competitors: [
+      ['dust'],
+      ['dutree', '--usage'],
+      ['du'],
+    ],
+  },
+  {
+    id: 'top-down',
+    pduCliArgs: ['--top-down'],
+    competitors: [
+      ['dust', '--apparent-size', '--reverse'],
+      ['dutree'],
+    ],
+  },
+  {
+    id: 'summary',
+    pduCliArgs: ['--max-depth=1'],
+    competitors: [
+      ['dutree', '--summary'],
+      ['du', '--apparent-size', '--total'],
+    ],
+  },
+  {
+    id: 'extreme-details',
+    pduCliArgs: ['--min-ratio=0'],
+    competitors: [
+      ['dutree'],
+      ['du', '--apparent-size'],
+    ],
+  },
+  {
+    id: 'no-sort',
+    pduCliArgs: ['--no-sort'],
+    competitors: [
+      ['du', '--apparent-size'],
+    ],
+  },
+  {
+    id: 'no-sort+summary',
+    pduCliArgs: ['--no-sort', '--max-depth=1'],
+    competitors: [
+      ['du', '--apparent-size', '--total'],
+    ],
+  },
+]