Merge branch 'master' into sd/speedup-applytheme

metrics/ttfp/run-benchmark.jl

@@ -25,8 +25,16 @@ base_branch = length(ARGS) > 2 ? ARGS[3] : "master"
 
 function run_benchmarks(projects; n=n_samples)
     benchmark_file = joinpath(@__DIR__, "benchmark-ttfp.jl")
-    # go A, B, A, B, A, etc.
-    for project in repeat(projects, n)
+    # go A, A, B, B, A, A, B, B, etc. because if A or B have some effect on their
+    # subsequent run, then we distribute those more evenly. If we used A, B, A, B then
+    # B would always influence A and A always B which might bias the results (something
+    # that can carry over separate processes like thermal throttling or so)
+
+    A, B = projects
+    As = Iterators.partition(fill(A, n), 2)
+    Bs = Iterators.partition(fill(B, n), 2)
+
+    for project in Iterators.flatten(Iterators.flatten(zip(As, Bs)))
         println(basename(project))
         run(`$(Base.julia_cmd()) --startup-file=no --project=$(project) $benchmark_file $Package`)
     end