From 625ad71984153c319288a1125aebbb7dc8cb1503 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Sun, 23 May 2021 16:47:58 +0700 Subject: [PATCH] ci: benchmark: Use TypeScript code to manage competing benchmark --- .github/workflows/benchmark.yaml | 63 -------------------------- ci/github-actions/benchmark.ts | 30 ++++++++++++- ci/github-actions/benchmark/matrix.ts | 65 +++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 64 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 71b2437..eae1d10 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -138,66 +138,3 @@ jobs: with: name: benchmark-reports path: tmp.benchmark-report.* - - - name: 'Benchmark: len' - run: | - commands=( - 'pdu --quantity=len tmp.sample' - 'dust --apparent-size tmp.sample' - 'dutree tmp.sample' - 'du --apparent-size tmp.sample' - ) - hyperfine --warmup=3 "${commands[@]}" - - - name: 'Benchmark: blksize' - run: | - commands=( - 'pdu --quantity=blksize tmp.sample' - 'dust tmp.sample' - 'dutree --usage tmp.sample' - 'du tmp.sample' - ) - hyperfine --warmup=3 "${commands[@]}" - - - name: 'Benchmark: top-down' - run: | - commands=( - 'pdu --top-down tmp.sample' - 'dust --apparent-size --reverse tmp.sample' - 'dutree tmp.sample' - ) - hyperfine --warmup=3 "${commands[@]}" - - - name: 'Benchmark: summary' - run: | - commands=( - 'pdu --max-depth=1 tmp.sample' - 'dutree --summary tmp.sample' - 'du --apparent-size --total tmp.sample' - ) - hyperfine --warmup=3 "${commands[@]}" - - - name: 'Benchmark: extreme details' - run: | - commands=( - 'pdu --min-ratio=0 tmp.sample' - 'dutree tmp.sample' - 'du --apparent-size tmp.sample' - ) - hyperfine --warmup=3 "${commands[@]}" - - - name: 'Benchmark: no sort' - run: | - commands=( - 'pdu --no-sort tmp.sample' - 'du --apparent-size tmp.sample' - ) - hyperfine --warmup=3 "${commands[@]}" - - - name: 'Benchmark: no sort, summary' - run: | - commands=( - 'pdu --no-sort --max-depth=1 tmp.sample' - 'du --apparent-size --total tmp.sample' - ) - hyperfine --warmup=3 "${commands[@]}" diff --git a/ci/github-actions/benchmark.ts b/ci/github-actions/benchmark.ts index a90ad6b..70b8f30 100644 --- a/ci/github-actions/benchmark.ts +++ b/ci/github-actions/benchmark.ts @@ -2,13 +2,25 @@ import console from 'console' import exec from 'exec-inline' import process from 'process' import shCmd from 'shell-escape' -import { SELF_BENCHMARK_MATRIX, SELF_BENCHMARK_HYPERFINE_NAMES, parseSelfBenchmarkCategory } from './benchmark/matrix' +import { + SELF_BENCHMARK_MATRIX, + SELF_BENCHMARK_HYPERFINE_NAMES, + COMPETING_BENCHMARK_MATRIX, + parseSelfBenchmarkCategory, +} from './benchmark/matrix' import * as reportFiles from './benchmark/report-files' import STRICT_BASH from './benchmark/strict-bash' const pduTargets = process.argv.slice(2) const errexit = (param: { readonly status: number | null }) => param.status !== 0 +function section(title: string) { + console.error(title) + console.error('='.repeat(title.length)) + console.error() +} + +section('Compare benchmark of pdu against other versions of itself') for (const { category, units } of SELF_BENCHMARK_MATRIX) { const { commandSuffix, reportName } = parseSelfBenchmarkCategory(category) console.error({ category, commandSuffix, reportName }) @@ -20,3 +32,19 @@ for (const { category, units } of SELF_BENCHMARK_MATRIX) { exec(...STRICT_BASH, '-c', shellCommand).exit(errexit) console.error() } + +section('Compare benchmark of pdu against its competitors') +for (const { id, pduCliArgs, competitors } of COMPETING_BENCHMARK_MATRIX) { + const commands = [ + `pdu ${pduCliArgs.join(' ')} tmp.sample`, + ...competitors.map(argv => `${argv.join(' ')} tmp.sample` as const), + ] as const + console.error({ id, commands }) + const reportName = `competing.${id}` as const + const exportReports = reportFiles.hyperfineArgs(reportName) + const commandLog = reportFiles.getFileName(reportName, 'log') + const hyperfineCommand = shCmd(['hyperfine', '--warmup=3', ...exportReports, ...commands]) + const shellCommand = `${hyperfineCommand} 2>&1 | tee ${commandLog}` + exec(...STRICT_BASH, '-c', shellCommand).exit(errexit) + console.error() +} diff --git a/ci/github-actions/benchmark/matrix.ts b/ci/github-actions/benchmark/matrix.ts index 55f15bb..e5123f5 100644 --- a/ci/github-actions/benchmark/matrix.ts +++ b/ci/github-actions/benchmark/matrix.ts @@ -74,3 +74,68 @@ export function getSelfBenchmarkHyperfineName( } export const SELF_BENCHMARK_HYPERFINE_NAMES = UNITS.map(getSelfBenchmarkHyperfineName) + +export interface CompetingBenchmarkCategory { + readonly id: string + readonly pduCliArgs: readonly string[] + readonly competitors: ReadonlyArray +} + +export const COMPETING_BENCHMARK_MATRIX: readonly CompetingBenchmarkCategory[] = [ + { + id: 'len', + pduCliArgs: ['--quantity=len'], + competitors: [ + ['dust', '--apparent-size'], + ['dutree'], + ['du', '--apparent-size'], + ], + }, + { + id: 'blksize', + pduCliArgs: ['--quantity=blksize'], + competitors: [ + ['dust'], + ['dutree', '--usage'], + ['du'], + ], + }, + { + id: 'top-down', + pduCliArgs: ['--top-down'], + competitors: [ + ['dust', '--apparent-size', '--reverse'], + ['dutree'], + ], + }, + { + id: 'summary', + pduCliArgs: ['--max-depth=1'], + competitors: [ + ['dutree', '--summary'], + ['du', '--apparent-size', '--total'], + ], + }, + { + id: 'extreme-details', + pduCliArgs: ['--min-ratio=0'], + competitors: [ + ['dutree'], + ['du', '--apparent-size'], + ], + }, + { + id: 'no-sort', + pduCliArgs: ['--no-sort'], + competitors: [ + ['du', '--apparent-size'], + ], + }, + { + id: 'no-sort+summary', + pduCliArgs: ['--no-sort', '--max-depth=1'], + competitors: [ + ['du', '--apparent-size', '--total'], + ], + }, +]