diff --git a/Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift b/Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift index 898063c2..ed067294 100644 --- a/Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift +++ b/Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift @@ -51,7 +51,7 @@ let help = --target Benchmark targets matching the regexp filter that should be run --skip-target Benchmark targets matching the regexp filter that should be skipped - --format The output format to use, default is 'text' (values: text, markdown, influx, jmh, histogramEncoded, histogram, histogramSamples, histogramPercentiles, metricP90AbsoluteThresholds) + --format The output format to use, default is 'text' (values: text, markdown, influx, jmh, jsonSmallerIsBetter, jsonBiggerIsBetter, histogramEncoded, histogram, histogramSamples, histogramPercentiles, metricP90AbsoluteThresholds) --metric Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks. (values: cpuUser, cpuSystem, cpuTotal, wallClock, throughput, peakMemoryResident, peakMemoryResidentDelta, peakMemoryVirtual, mallocCountSmall, mallocCountLarge, mallocCountTotal, allocatedResidentMemory, memoryLeaked, syscalls, contextSwitches, threads, threadsRunning, readSyscalls, writeSyscalls, readBytesLogical, writeBytesLogical, readBytesPhysical, writeBytesPhysical, instructions, retainCount, releaseCount, retainReleaseDelta, custom) diff --git a/Plugins/BenchmarkCommandPlugin/Command+Helpers.swift b/Plugins/BenchmarkCommandPlugin/Command+Helpers.swift index 45f37bf5..f46ecee5 100644 --- a/Plugins/BenchmarkCommandPlugin/Command+Helpers.swift +++ b/Plugins/BenchmarkCommandPlugin/Command+Helpers.swift @@ -34,6 +34,10 @@ public enum OutputFormat: String, CaseIterable { case influx /// JMH format consumable by http://jmh.morethan.io case jmh + /// JSON format consumable by https://github.com/benchmark-action/github-action-benchmark + case jsonSmallerIsBetter + /// JSON format consumable by https://github.com/benchmark-action/github-action-benchmark + case jsonBiggerIsBetter /// The encoded representation of the underlying histograms capturing the benchmark data, for programmatic use (Codable). case histogramEncoded /// The histogram percentiles, average, deviation, sample count etc in standard HDR Histogram text format consumable by http://hdrhistogram.github.io/HdrHistogram/plotFiles.html diff --git a/Plugins/BenchmarkTool/BenchmarkTool+Export+JSON.swift b/Plugins/BenchmarkTool/BenchmarkTool+Export+JSON.swift new file mode 100644 index 00000000..650c34fd --- /dev/null +++ b/Plugins/BenchmarkTool/BenchmarkTool+Export+JSON.swift @@ -0,0 +1,67 @@ +import Benchmark +import Foundation + +struct BenchmarkJSON: Codable { + var name: String + var unit: String + var value: Double + var range: String? + var extra: String? +} + +extension BenchmarkTool { + func convertToJSON(_ baseline: BenchmarkBaseline, polarity: BenchmarkMetric.Polarity = .prefersSmaller) throws -> String { + var allBenchmarks: [BenchmarkJSON] = [] + + baseline.targets.forEach { target in + var keys = baseline.results.keys.sorted(by: { $0.name < $1.name }) + keys.removeAll(where: { $0.target != target }) + + keys.forEach { test in + if let results = baseline.results[test] { + results.forEach { result in + if polarity != result.metric.polarity { + return + } + + let testName = test.name + let metricName = result.metric.description + let fullName = "\(testName) - \(metricName)" + + let percentiles = result.statistics.percentiles(for: [80.0, 95.0, 99.0]).map { result.scale($0) } + let (p80, p95, p99) = (percentiles[0], percentiles[1], percentiles[2]) + + var range: String? + var extra: String? + + let stdDeviation = result.statistics.histogram.stdDeviation + if stdDeviation > 0 { + //range = String(format: "± %.3f", stdDeviation) + if p80 != p95 { + extra = "80th percentile: \(p80)\n95th percentile: \(p95)" + } + } + + let unit = result.metric.countable + ? Statistics.Units(result.timeUnits).description + : result.timeUnits.description + + allBenchmarks.append(BenchmarkJSON( + name: fullName, + unit: unit, + value: Double(p99), + range: range, + extra: extra + )) + } + } + } + } + + let encoder = JSONEncoder() + encoder.outputFormatting = .prettyPrinted + let jsonData = try encoder.encode(allBenchmarks) + return String(data: jsonData, encoding: .utf8) ?? "[]" + } + +} diff --git a/Plugins/BenchmarkTool/BenchmarkTool+Export.swift b/Plugins/BenchmarkTool/BenchmarkTool+Export.swift index 48802577..db376a2a 100644 --- a/Plugins/BenchmarkTool/BenchmarkTool+Export.swift +++ b/Plugins/BenchmarkTool/BenchmarkTool+Export.swift @@ -276,6 +276,16 @@ extension BenchmarkTool { print("Failed to encode json for \(outputResults)") } } + case .jsonSmallerIsBetter: + try write( + exportData: "\(convertToJSON(baseline, polarity: .prefersSmaller))", + fileName: cleanupStringForShellSafety("\(baselineName).json") + ) + case .jsonBiggerIsBetter: + try write( + exportData: "\(convertToJSON(baseline, polarity: .prefersLarger))", + fileName: cleanupStringForShellSafety("\(baselineName)-bigger-is-better.json") + ) } } diff --git a/Sources/BenchmarkShared/Command+Helpers.swift b/Sources/BenchmarkShared/Command+Helpers.swift index 1b44b341..81fa48e9 100644 --- a/Sources/BenchmarkShared/Command+Helpers.swift +++ b/Sources/BenchmarkShared/Command+Helpers.swift @@ -30,6 +30,10 @@ public enum OutputFormat: String, CaseIterable { case influx /// JMH format consumable by http://jmh.morethan.io case jmh + /// JSON format consumable by https://github.com/benchmark-action/github-action-benchmark + case jsonSmallerIsBetter + /// JSON format consumable by https://github.com/benchmark-action/github-action-benchmark + case jsonBiggerIsBetter /// The encoded representation of the underlying histograms capturing the benchmark data, for programmatic use (Codable). case histogramEncoded /// The histogram percentiles, average, deviation, sample count etc in standard HDR Histogram text format consumable by http://hdrhistogram.github.io/HdrHistogram/plotFiles.html