Skip to content

Commit e59009d

Browse files
authored
Align benchmark scaling and minimum samples (#2839)
# Motivation We want to have an aligned strategy across all of our benchmarks in how we use the scaling factor and samples. # Modification This PR makes sure that any inner loop is scaled by the scaling factor. Moreover, we want to run more than 1 iteration. To do this we set an almost infinite maximum duration and a maximum iteration count. # Result More consistent benchmarks
1 parent 286834d commit e59009d

16 files changed

+34
-27
lines changed

Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift

+6-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,12 @@ let benchmarks = {
2323

2424
Benchmark(
2525
"NIOAsyncChannel.init",
26-
configuration: Benchmark.Configuration(metrics: defaultMetrics)
26+
configuration: .init(
27+
metrics: defaultMetrics,
28+
scalingFactor: .kilo,
29+
maxDuration: .seconds(10_000_000),
30+
maxIterations: 10
31+
)
2732
) { benchmark in
2833
// Elide the cost of the 'EmbeddedChannel'. It's only used for its pipeline.
2934
var channels: [EmbeddedChannel] = []

Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift

+5-3
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ let benchmarks = {
2626
"TCPEcho",
2727
configuration: .init(
2828
metrics: defaultMetrics,
29-
timeUnits: .milliseconds,
30-
scalingFactor: .mega
29+
scalingFactor: .mega,
30+
maxDuration: .seconds(10_000_000),
31+
maxIterations: 5
3132
)
3233
) { benchmark in
3334
try runTCPEcho(
@@ -43,8 +44,9 @@ let benchmarks = {
4344
"TCPEchoAsyncChannel",
4445
configuration: .init(
4546
metrics: defaultMetrics,
46-
timeUnits: .milliseconds,
4747
scalingFactor: .mega,
48+
maxDuration: .seconds(10_000_000),
49+
maxIterations: 5,
4850
// We are expecting a bit of allocation variance due to an allocation
4951
// in the Concurrency runtime which happens when resuming a continuation.
5052
thresholds: [.mallocCountTotal: .init(absolute: [.p90: 2000])],
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 8
3-
}
2+
"mallocCountTotal" : 8000
3+
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 108
2+
"mallocCountTotal" : 555
33
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 165000
3-
}
2+
"mallocCountTotal" : 164375
3+
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 8
3-
}
2+
"mallocCountTotal" : 8000
3+
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 110
2+
"mallocCountTotal" : 556
33
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 8
3-
}
2+
"mallocCountTotal" : 8000
3+
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 110
2+
"mallocCountTotal" : 556
33
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 164000
3-
}
2+
"mallocCountTotal" : 164381
3+
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 8
3-
}
2+
"mallocCountTotal" : 8000
3+
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 108
2+
"mallocCountTotal" : 548
33
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 165000
3-
}
2+
"mallocCountTotal" : 164376
3+
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 8
3-
}
2+
"mallocCountTotal" : 8000
3+
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 108
2+
"mallocCountTotal" : 548
33
}
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"mallocCountTotal" : 165000
3-
}
2+
"mallocCountTotal" : 164371
3+
}

0 commit comments

Comments
 (0)