feat(patch): Export results in json format readable by github-action-benchmark

Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift

@@ -51,7 +51,7 @@ let help =
     --target <target>       Benchmark targets matching the regexp filter that should be run
     --skip-target <skip-target>
                           Benchmark targets matching the regexp filter that should be skipped
-    --format <format>       The output format to use, default is 'text' (values: text, markdown, influx, jmh, histogramEncoded, histogram, histogramSamples, histogramPercentiles, metricP90AbsoluteThresholds)
+    --format <format>       The output format to use, default is 'text' (values: text, markdown, influx, jmh, jsonSmallerIsBetter, jsonBiggerIsBetter, histogramEncoded, histogram, histogramSamples, histogramPercentiles, metricP90AbsoluteThresholds)
     --metric <metric>       Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks. (values: cpuUser, cpuSystem, cpuTotal, wallClock, throughput,
                           peakMemoryResident, peakMemoryResidentDelta, peakMemoryVirtual, mallocCountSmall, mallocCountLarge, mallocCountTotal, allocatedResidentMemory, memoryLeaked, syscalls, contextSwitches, threads,
                           threadsRunning, readSyscalls, writeSyscalls, readBytesLogical, writeBytesLogical, readBytesPhysical, writeBytesPhysical, instructions, retainCount, releaseCount, retainReleaseDelta, custom)

Plugins/BenchmarkCommandPlugin/Command+Helpers.swift

@@ -34,6 +34,10 @@ public enum OutputFormat: String, CaseIterable {
     case influx
     /// JMH format consumable by http://jmh.morethan.io
     case jmh
+    /// JSON format consumable by https://github.com/benchmark-action/github-action-benchmark
+    case jsonSmallerIsBetter
+    /// JSON format consumable by https://github.com/benchmark-action/github-action-benchmark
+    case jsonBiggerIsBetter
     /// The encoded representation of the underlying histograms capturing the benchmark data, for programmatic use (Codable).
     case histogramEncoded
     /// The histogram percentiles, average, deviation, sample count etc in standard HDR Histogram text format consumable by http://hdrhistogram.github.io/HdrHistogram/plotFiles.html

Plugins/BenchmarkTool/BenchmarkTool+Export+JSON.swift

@@ -0,0 +1,67 @@
+import Benchmark
+import Foundation
+
+struct BenchmarkJSON: Codable {
+    var name: String
+    var unit: String
+    var value: Double
+    var range: String?
+    var extra: String?
+}
+
+extension BenchmarkTool {
+    func convertToJSON(_ baseline: BenchmarkBaseline, polarity: BenchmarkMetric.Polarity = .prefersSmaller) throws -> String {
+        var allBenchmarks: [BenchmarkJSON] = []
+
+        baseline.targets.forEach { target in
+            var keys = baseline.results.keys.sorted(by: { $0.name < $1.name })
+            keys.removeAll(where: { $0.target != target })
+
+            keys.forEach { test in
+                if let results = baseline.results[test] {
+                    results.forEach { result in
+                        if polarity != result.metric.polarity {
+                            return
+                        }
+
+                        let testName = test.name
+                        let metricName = result.metric.description
+                        let fullName = "\(testName) - \(metricName)"
+
+                        let percentiles = result.statistics.percentiles(for: [80.0, 95.0, 99.0]).map { result.scale($0) }
+                        let (p80, p95, p99) = (percentiles[0], percentiles[1], percentiles[2])
+
+                        var range: String?
+                        var extra: String?
+
+                        let stdDeviation = result.statistics.histogram.stdDeviation
+                        if stdDeviation > 0 {
+                            //range = String(format: "± %.3f", stdDeviation)
+                            if p80 != p95 {
+                                extra = "80th percentile: \(p80)\n95th percentile: \(p95)"
+                            }
+                        }
+
+                        let unit = result.metric.countable
+                            ? Statistics.Units(result.timeUnits).description
+                            : result.timeUnits.description
+
+                        allBenchmarks.append(BenchmarkJSON(
+                            name: fullName,
+                            unit: unit,
+                            value: Double(p99),
+                            range: range,
+                            extra: extra
+                        ))
+                    }
+                }
+            }
+        }
+
+        let encoder = JSONEncoder()
+        encoder.outputFormatting = .prettyPrinted
+        let jsonData = try encoder.encode(allBenchmarks)
+        return String(data: jsonData, encoding: .utf8) ?? "[]"
+    }
+
+}

Plugins/BenchmarkTool/BenchmarkTool+Export.swift

@@ -276,6 +276,16 @@ extension BenchmarkTool {
                     print("Failed to encode json for \(outputResults)")
                 }
             }
+        case .jsonSmallerIsBetter:
+            try write(
+                exportData: "\(convertToJSON(baseline, polarity: .prefersSmaller))",
+                fileName: cleanupStringForShellSafety("\(baselineName).json")
+            )
+        case .jsonBiggerIsBetter:
+            try write(
+                exportData: "\(convertToJSON(baseline, polarity: .prefersLarger))",
+                fileName: cleanupStringForShellSafety("\(baselineName)-bigger-is-better.json")
+            )
         }
     }
 

Sources/BenchmarkShared/Command+Helpers.swift

@@ -30,6 +30,10 @@ public enum OutputFormat: String, CaseIterable {
     case influx
     /// JMH format consumable by http://jmh.morethan.io
     case jmh
+    /// JSON format consumable by https://github.com/benchmark-action/github-action-benchmark
+    case jsonSmallerIsBetter
+    /// JSON format consumable by https://github.com/benchmark-action/github-action-benchmark
+    case jsonBiggerIsBetter
     /// The encoded representation of the underlying histograms capturing the benchmark data, for programmatic use (Codable).
     case histogramEncoded
     /// The histogram percentiles, average, deviation, sample count etc in standard HDR Histogram text format consumable by http://hdrhistogram.github.io/HdrHistogram/plotFiles.html