Add a GitHub action for continuous benchmarking

We've put in a lot of effort into making Dex faster in the recent months
and it would be a waste to have it all slip away. So, to prevent that,
I've set up a simple system for continuous performance tracking. This
includes three parts:
1. I created an orphan branch named `performance-data`, which we'll use
as a simple CSV-based database of performance measurements. I ran a few
benchmarks using the new `benchmarks/continuous.py` script to
pre-populate it.
2. On the `gh-pages` branch, there's a new `performance.html` file,
now accessible via [this URL](https://google-research.github.io/dex-lang/performance.html).
It pulls the data from the `performance-data` branch and displays it as
a series of plots showing changes in total allocation and end-to-end
execution times of a few of our examples.
3. This commit checks in the file used to perform benchmarks, and adds a
GitHub action that will run it every time we push to `main`.

While total allocation numbers are very stable, at this point you might
be worried that any time estimates we get from the free VMs that execute
GitHub actions will be too noisy to provide any good signal. And if we
did it naively, you wouldn't be wrong! To make the drift between
different machines lower we use the techniques outlined in
[this post](https://labs.quansight.org/blog/2021/08/github-actions-benchmarks/).
The idea is to report _relative change_ in execution time compared to
baseline. Most importantly, any time we want to evaluate a new commit,
_we rebenchmark the baseline_ to adjust for the differences between
machines. To minimize noise on shorter time scales, we interleave the
evaluations of baseline compiler with those using newer versions.

Author: apaszke

.github/workflows/bench.yaml

@@ -0,0 +1,58 @@
+name: Continuous benchmarking
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+        include:
+          - os: ubuntu-20.04
+            install_deps: sudo apt-get install llvm-12-tools llvm-12-dev pkg-config
+            path_extension: /usr/lib/llvm-12/bin
+
+    steps:
+    - name: Checkout the repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Install system dependencies
+      run: |
+        ${{ matrix.install_deps }}
+        echo "${{ matrix.path_extension }}" >> $GITHUB_PATH
+
+    - name: Cache
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.stack
+
+        key: ${{ runner.os }}-bench-v1-${{ hashFiles('**/*.cabal', 'stack*.yaml') }}
+        restore-keys: ${{ runner.os }}-bench-v1
+
+    - name: Benchmark
+      run: python3 benchmarks/continuous.py /tmp/new-perf-data.csv /tmp/new-commits.csv ${GITHUB_SHA}
+
+    - name: Switch to the data branch
+      uses: actions/checkout@v2
+      with:
+        ref: performance-data
+
+    - name: Append new data points
+      run: |
+        cat /tmp/new-perf-data.csv >>performance.csv
+        cat /tmp/new-commits.csv >>commits.csv
+
+    - name: Commit new data points
+      run: |
+          git config --global user.name 'Dex CI'
+          git config --global user.email '[email protected]'
+          git add performance.csv commits.csv
+          git commit -m "Add measurements for ${GITHUB_SHA}"
+          git push

.github/workflows/docs.yaml

@@ -20,13 +20,6 @@ jobs:
     - name: Checkout the repository
       uses: actions/checkout@v2
 
-    - name: Setup Haskell Stack
-      uses: actions/setup-haskell@v1
-      with:
-        enable-stack: true
-        stack-no-global: true
-        stack-version: 'latest'
-
     - name: Install system dependencies
       run: |
         ${{ matrix.install_deps }}

benchmarks/continuous.py

@@ -0,0 +1,117 @@
+import re
+import os
+import sys
+import csv
+import subprocess
+import tempfile
+from functools import partial
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Union, Sequence
+
+
+BASELINE = '8dd1aa8539060a511d0f85779ae2c8019162f567'
+BENCH_EXAMPLES = [('kernelregression', 10), ('psd', 10), ('fluidsim', 10), ('regression', 10)]
+
+
+def run(*args, capture=False, env=None):
+  print('> ' + ' '.join(map(str, args)))
+  return subprocess.run(args, check=True, text=True, capture_output=capture, env=env)
+
+
+def read(*args, **kwargs):
+  return run(*args, capture=True, **kwargs).stdout
+
+
+def read_stderr(*args, **kwargs):
+  return run(*args, capture=True, **kwargs).stderr
+
+
+def build(commit):
+  if os.path.exists(commit):
+    print(f'Skipping the build of {commit}')
+  else:
+    run('git', 'checkout', commit)
+    run('make', 'install', env=dict(os.environ, PREFIX=commit))
+  dex_bin = Path.cwd() / commit / 'dex'
+  return dex_bin
+
+
+def benchmark(baseline_bin, latest_bin):
+  with tempfile.TemporaryDirectory() as tmp:
+    def clean(bin, uniq):
+      run(bin, 'clean', env={'XDG_CACHE_HOME': Path(tmp) / uniq})
+    def bench(bin, uniq, bench_name, path):
+      return parse_result(
+          read_stderr(bin, 'script', path, '+RTS', '-s',
+                      env={'XDG_CACHE_HOME': Path(tmp) / uniq}))
+    baseline_clean = partial(clean, baseline_bin, 'baseline')
+    baseline_bench = partial(bench, baseline_bin, 'baseline')
+    latest_clean = partial(clean, latest_bin, 'latest')
+    latest_bench = partial(bench, latest_bin, 'latest')
+    results = []
+    for example, repeats in BENCH_EXAMPLES:
+      path = Path('examples') / (example + '.dx')
+      # warm-up the caches
+      baseline_clean()
+      baseline_bench(example, path)
+      latest_clean()
+      latest_bench(example, path)
+      for i in range(repeats):
+        print(f'Iteration {i}')
+        baseline_alloc, baseline_time = baseline_bench(example, path)
+        latest_alloc, latest_time = latest_bench(example, path)
+        print(baseline_alloc, '->', latest_alloc)
+        print(baseline_time, '->', latest_time)
+        results.append(Result(example, 'alloc', latest_alloc))
+        results.append(Result(example, 'time_rel', latest_time / baseline_time))
+    return results
+
+
+@dataclass
+class Result:
+  benchmark: str
+  measure: str
+  value: Union[int, float]
+
+
+ALLOC_PATTERN = re.compile(r"^\s*([0-9,]+) bytes allocated in the heap", re.M)
+TIME_PATTERN = re.compile(r"^\s*Total\s*time\s*([0-9.]+)s", re.M)
+def parse_result(output):
+  alloc_line = ALLOC_PATTERN.search(output)
+  if alloc_line is None:
+    raise RuntimeError("Couldn't extract total allocations")
+  total_alloc = int(alloc_line.group(1).replace(',', ''))
+  time_line = TIME_PATTERN.search(output)
+  if time_line is None:
+    raise RuntimeError("Couldn't extract total time")
+  total_time = float(time_line.group(1))
+  return total_alloc, total_time
+
+
+def save(commit, results: Sequence[Result], datapath, commitpath):
+  with open(datapath, 'a', newline='') as datafile:
+    writer = csv.writer(datafile, delimiter=',', quotechar='"', dialect='unix')
+    for r in results:
+      writer.writerow((commit, r.benchmark, r.measure, r.value))
+  with open(commitpath, 'a', newline='') as commitfile:
+    writer = csv.writer(commitfile, delimiter=',', quotechar='"', dialect='unix')
+    date = read('git', 'show', '-s', '--format=%ct', commit, '--').strip()
+    writer.writerow([commit, date])
+
+
+def main(argv):
+  if len(argv) != 3:
+    raise ValueError("Expected three arguments!")
+  datapath, commitpath, commit = argv
+  print('Building baseline: {BASELINE}')
+  baseline_bin = build(BASELINE)
+  print(f'Building latest: {commit}')
+  latest_bin = build(commit)
+  results = benchmark(baseline_bin, latest_bin)
+  save(commit, results, datapath, commitpath)
+  print('DONE!')
+
+
+if __name__ == '__main__':
+  main(sys.argv[1:])