From b8a016fc577a3b8ee310caf4683def2681851e04 Mon Sep 17 00:00:00 2001 From: Jonathan Cubides Date: Fri, 1 Dec 2023 18:39:39 +0100 Subject: [PATCH] Add Makefile to hyperfine benchmarks (#2533) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds a target for our Makefile to run benchmarks using hyperfine. We run different commands which can be easily modified in bench/hyperfine/Makefile. ``` make hyperfine-benchmarks ``` After running this on your terminal, checkout the generated file bench/hyperfine/README.md for the results. The results below are the runs using the following module. ``` module fibo; import Stdlib.Prelude open; fib : Nat → Nat → Nat → Nat | zero x1 _ := x1 | (suc n) x1 x2 := fib n x2 (x1 + x2); fibonacci (n : Nat) : Nat := fib n 0 1; main : IO := printNatLn (fibonacci 50); ``` (For now, I got the following on my machine (ran in about 5min or less) Darwin Jonathans-MacBook-Pro-2.local 22.6.0 Darwin Kernel Version 22.6.0: Fri Sep 15 13:41:28 PDT 2023; root:xnu-8796.141.3.700.8~1/RELEASE_ARM64_T6020 arm64 arm) - The binary without the version below is the latest commit on main. # Hyperfine Benchmarks ## dev parse | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `juvix-v0.4.3 dev parse fibo.juvix` | 184.3 ± 5.4 | 178.5 | 193.2 | 1.06 ± 0.04 | | `juvix-v0.5.0 dev parse fibo.juvix` | 184.7 ± 6.9 | 179.2 | 201.3 | 1.06 ± 0.05 | | `juvix-v0.5.1 dev parse fibo.juvix` | 174.2 ± 5.4 | 167.9 | 181.0 | 1.00 | | `juvix-v0.5.2 dev parse fibo.juvix` | 181.3 ± 1.6 | 179.5 | 184.1 | 1.04 ± 0.03 | | `juvix-v0.5.3 dev parse fibo.juvix` | 1185.8 ± 5.7 | 1178.4 | 1197.3 | 6.81 ± 0.21 | | `juvix-v0.5.4 dev parse fibo.juvix` | 1308.4 ± 6.9 | 1297.6 | 1319.3 | 7.51 ± 0.23 | | `juvix dev parse fibo.juvix` | 1311.0 ± 5.5 | 1303.2 | 1318.5 | 7.53 ± 0.23 | ## dev highlight | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `juvix-v0.4.3 dev highlight fibo.juvix` | 770.2 ± 67.5 | 742.9 | 961.6 | 1.01 ± 0.09 | | `juvix-v0.5.0 dev highlight fibo.juvix` | 762.7 ± 11.8 | 749.9 | 787.2 | 1.00 | | `juvix-v0.5.1 dev highlight fibo.juvix` | 849.3 ± 9.3 | 836.5 | 863.9 | 1.11 ± 0.02 | | `juvix-v0.5.2 dev highlight fibo.juvix` | 873.5 ± 21.1 | 855.2 | 918.9 | 1.15 ± 0.03 | | `juvix-v0.5.3 dev highlight fibo.juvix` | 2035.9 ± 69.5 | 1946.9 | 2125.4 | 2.67 ± 0.10 | | `juvix-v0.5.4 dev highlight fibo.juvix` | 2218.0 ± 57.3 | 2169.5 | 2316.4 | 2.91 ± 0.09 | | `juvix dev highlight fibo.juvix` | 2206.5 ± 55.9 | 2150.4 | 2296.6 | 2.89 ± 0.09 | ## typecheck | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `juvix-v0.4.3 typecheck fibo.juvix` | 684.1 ± 5.1 | 674.1 | 691.0 | 1.00 | | `juvix-v0.5.0 typecheck fibo.juvix` | 695.9 ± 14.2 | 679.0 | 720.6 | 1.02 ± 0.02 | | `juvix-v0.5.1 typecheck fibo.juvix` | 808.5 ± 18.2 | 780.8 | 848.6 | 1.18 ± 0.03 | | `juvix-v0.5.2 typecheck fibo.juvix` | 816.0 ± 13.9 | 802.3 | 846.3 | 1.19 ± 0.02 | | `juvix-v0.5.3 typecheck fibo.juvix` | 1934.3 ± 29.0 | 1907.3 | 1992.1 | 2.83 ± 0.05 | | `juvix-v0.5.4 typecheck fibo.juvix` | 2106.8 ± 19.6 | 2075.6 | 2138.9 | 3.08 ± 0.04 | | `juvix typecheck fibo.juvix` | 2135.3 ± 29.9 | 2107.4 | 2183.2 | 3.12 ± 0.05 | ## compile -o /dev/null | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `juvix-v0.4.3 compile -o /dev/null fibo.juvix` | 754.3 ± 4.9 | 745.2 | 759.8 | 1.00 ± 0.01 | | `juvix-v0.5.0 compile -o /dev/null fibo.juvix` | 752.6 ± 3.9 | 745.2 | 757.6 | 1.00 | | `juvix-v0.5.1 compile -o /dev/null fibo.juvix` | 845.4 ± 5.7 | 837.6 | 857.0 | 1.12 ± 0.01 | | `juvix-v0.5.2 compile -o /dev/null fibo.juvix` | 883.7 ± 15.7 | 864.5 | 918.3 | 1.17 ± 0.02 | | `juvix-v0.5.3 compile -o /dev/null fibo.juvix` | 1990.8 ± 12.5 | 1975.0 | 2010.4 | 2.65 ± 0.02 | | `juvix-v0.5.4 compile -o /dev/null fibo.juvix` | 2193.9 ± 5.6 | 2182.7 | 2200.5 | 2.91 ± 0.02 | | `juvix compile -o /dev/null fibo.juvix` | 2197.4 ± 11.6 | 2185.0 | 2226.0 | 2.92 ± 0.02 | ## eval | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `juvix-v0.4.3 eval fibo.juvix` | 680.9 ± 5.4 | 669.7 | 687.1 | 1.00 | | `juvix-v0.5.0 eval fibo.juvix` | 681.7 ± 5.0 | 675.0 | 689.8 | 1.00 ± 0.01 | | `juvix-v0.5.1 eval fibo.juvix` | 772.7 ± 2.7 | 769.0 | 777.9 | 1.13 ± 0.01 | | `juvix-v0.5.2 eval fibo.juvix` | 796.1 ± 2.7 | 791.9 | 799.5 | 1.17 ± 0.01 | | `juvix-v0.5.3 eval fibo.juvix` | 1902.2 ± 6.5 | 1889.6 | 1913.2 | 2.79 ± 0.02 | | `juvix-v0.5.4 eval fibo.juvix` | 2101.3 ± 5.9 | 2093.1 | 2112.1 | 3.09 ± 0.03 | | `juvix eval fibo.juvix` | 2111.1 ± 17.4 | 2085.5 | 2148.3 | 3.10 ± 0.04 | --------- Co-authored-by: Paul Cadman --- Makefile | 7 ++++++ bench/hyperfine/.gitignore | 3 +++ bench/hyperfine/Makefile | 46 ++++++++++++++++++++++++++++++++++++++ bench/hyperfine/fibo.juvix | 11 +++++++++ 4 files changed, 67 insertions(+) create mode 100644 bench/hyperfine/.gitignore create mode 100644 bench/hyperfine/Makefile create mode 100644 bench/hyperfine/fibo.juvix diff --git a/Makefile b/Makefile index 6f576181d5..d080d89819 100644 --- a/Makefile +++ b/Makefile @@ -257,3 +257,10 @@ smoke: install submodules .PHONY : changelog changelog : @github_changelog_generator + +HYPERFINEBIN := $(shell command -v hyperfine 2> /dev/null) + +.PHONY : hyperfine-benchmarks +hyperfine-benchmarks: + @$(if $(HYPERFINEBIN),, $(error "hyperfine not found, please install it using cargo or from https://github.com/sharkdp/hyperfine")) + cd bench/hyperfine && ${MAKE} diff --git a/bench/hyperfine/.gitignore b/bench/hyperfine/.gitignore new file mode 100644 index 0000000000..a646ec39db --- /dev/null +++ b/bench/hyperfine/.gitignore @@ -0,0 +1,3 @@ +*.md +*.json +*.png diff --git a/bench/hyperfine/Makefile b/bench/hyperfine/Makefile new file mode 100644 index 0000000000..28052d13d7 --- /dev/null +++ b/bench/hyperfine/Makefile @@ -0,0 +1,46 @@ +HELL := /bin/bash +HYPERFINE?=hyperfine +WARMUP?=2 +RUNS?=10 +HYPERFINEFLAGS?=--warmup ${WARMUP} \ + --runs ${RUNS} + +JUVIXVERSIONS?=-v0.4.3,-v0.5.0,-v0.5.1,-v0.5.2,-v0.5.3,-v0.5.4, +TASKS?="dev root" \ + "dev parse" \ + "dev highlight" \ + typecheck \ + "compile -o /dev/null" \ + "compile -o /dev/null -t wasm32-wasi" \ + "compile -o /dev/null -t core" \ + "compile -o /dev/null -t asm" \ + eval + +GLOBALOPTS?= + +FILENAME?=fibo.juvix + +all: hyperfine + +.PHONY: hyperfine +hyperfine: + @echo "# Hyperfine Benchmarks" > README.md + @for task in ${TASKS}; do \ + TASK=$$task ${MAKE} run; \ + done; + +.PHONY: run +run: + @${HYPERFINE} ${HYPERFINEFLAGS} \ + --parameter-list version ${JUVIXVERSIONS} \ + 'juvix{version} ${TASK} ${FILENAME} ${GLOBALOPTS}' \ + --export-markdown TMP.md + @echo "" >> README.md + @echo "## ${TASK}" >> README.md + @echo "" >> README.md + @cat TMP.md >> README.md + rm -f TMP.md + +.PHONY: clean +clean: + @rm -rf *.json *.md diff --git a/bench/hyperfine/fibo.juvix b/bench/hyperfine/fibo.juvix new file mode 100644 index 0000000000..174174cf9f --- /dev/null +++ b/bench/hyperfine/fibo.juvix @@ -0,0 +1,11 @@ +module fibo; + +import Stdlib.Prelude open; + +fib : Nat → Nat → Nat → Nat + | zero x1 _ := x1 + | (suc n) x1 x2 := fib n x2 (x1 + x2); + +fibonacci (n : Nat) : Nat := fib n 0 1; + +main : IO := printNatLn (fibonacci 50);