diff --git a/benchmark_automation/benchmark_automation.cs b/benchmark_automation/benchmark_automation.cs
index 12a2d60f77..f3ee10971c 100644
--- a/benchmark_automation/benchmark_automation.cs
+++ b/benchmark_automation/benchmark_automation.cs
@@ -122,11 +122,12 @@ private static void Main(string[] args) {
     mathematica_stream.WriteLine("}");
     mathematica_stream.Close();
     var unseen_benchmarks = (from seen in seen_benchmarks
-                             where seen.Value
+                             where !seen.Value
                              select seen.Key).ToArray();
     if (unseen_benchmarks.Length > 0 ) {
       throw new ArgumentException(
-          "The following benchmarks were not run: " + unseen_benchmarks);
+          "The following benchmarks were not run:\n" +
+          string.Join("\n", unseen_benchmarks));
     }
   }