Skip to content

Commit 45bae90

Browse files
ptoffyMahdiBM
ptoffy
and
MahdiBM
authored
Add benchmarks (#103)
* Start adding benchmarks * Remove approval requirement for now * Update workflow * Update workflow * Add more benchmarks * Update benchmark CI with the jwt-kit one * [CI] ci to get triggered on every file change at in Benchmarks folder * Try RunsOn magic cache * configure thresholds * remove s3-cache from this specific ci file * Create runs-on.yml * update thresholds * move runs-on configuration step * get thresholds working * update machine name * Update machine name * use `cpuUser` instead of the other cpu metrics * update benchmarks * Fix benchmark code * rename machine * configure RunsOn first * adjust benchmarks * fix threshold files names * do 1000x for cpu time * fix ranges * revert some of the iteration changes * aesthetical rename * Try better CPU benchmarks * use 1000 scale factor for cpu benchmarks * try 100x with manual * refinements * add tolerances * fix tolerance * SerializerCPUTime 10x -> 100x * quote branch names for clarity * update thresholds * resolve nit picks @gwynne is going to make * increase iterations for more stability * increase max duration * better thresholds and iterations * more adjustments * allocation benchmarks to only run once * increase big message size to what it should be * decrease rounds * minor refinements * try something weird * try different method * Use `unsafeUninitializedCapacity` initialiser * Revert "Use `unsafeUninitializedCapacity` initialiser" This reverts commit a89c638. * Try something * minor fixes * minor * refinements * smaller chunk * use array * more fixes * minor fixes * refinements * just use AsyncStream * better thresholds * more benchs + revert to `AsyncSyncStream` for max accuracy * refinements * more benchmarks and refinements * fixes * adjust thresholds * minor refinement * thresholds * thresholds * thresholds * increase iterations * thresholds * thresholds * minor * better message * thresholds * minor * fix * better detect PR events * Try thollander/actions-comment-pull-request * minor --------- Co-authored-by: MahdiBM <[email protected]>
1 parent 74a5a66 commit 45bae90

20 files changed

+636
-0
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
name: benchmark
2+
on:
3+
workflow_dispatch:
4+
pull_request_review:
5+
types: [submitted]
6+
pull_request:
7+
branches: [main]
8+
types: [synchronize]
9+
paths:
10+
- Sources/*.swift
11+
- Benchmarks/
12+
- .github/workflows/benchmark.yml
13+
14+
jobs:
15+
benchmark-vs-thresholds:
16+
# Run the job only if it's a manual workflow dispatch, or if this event is a pull-request approval event, or if someone has rerun the job.
17+
if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved' || github.run_attempt > 1
18+
19+
# https://runs-on.com/features/custom-runners/
20+
runs-on:
21+
labels:
22+
- runs-on
23+
- runner=2cpu-4ram
24+
- run-id=${{ github.run_id }}
25+
26+
container: swift:noble
27+
28+
defaults:
29+
run:
30+
shell: bash
31+
32+
env:
33+
PR_COMMENT: null # will be populated later
34+
35+
steps:
36+
- name: Configure RunsOn
37+
uses: runs-on/action@v1
38+
39+
- name: Check out code
40+
uses: actions/checkout@v4
41+
with:
42+
fetch-depth: 0
43+
44+
- name: Configure git
45+
run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"
46+
47+
# jemalloc is a dependency of the Benchmarking package
48+
# actions/cache will detect zstd and will become much faster.
49+
- name: Install jemalloc, curl, jq and zstd
50+
run: |
51+
set -eu
52+
53+
apt-get update -y
54+
apt-get install -y libjemalloc-dev curl jq zstd
55+
56+
- name: Restore .build
57+
id: restore-cache
58+
uses: actions/cache/restore@v4
59+
with:
60+
path: Benchmarks/.build
61+
key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.event.pull_request.base.sha || github.event.after }}"
62+
restore-keys: "swiftpm-benchmark-build-${{ runner.os }}-"
63+
64+
- name: Run benchmarks for branch '${{ github.head_ref || github.ref_name }}'
65+
run: |
66+
swift package -c release --disable-sandbox \
67+
--package-path Benchmarks \
68+
benchmark baseline update \
69+
'${{ github.head_ref || github.ref_name }}'
70+
71+
- name: Read benchmark result
72+
id: read-benchmark
73+
run: |
74+
set -eu
75+
76+
swift package -c release --disable-sandbox \
77+
--package-path Benchmarks \
78+
benchmark baseline read \
79+
'${{ github.head_ref || github.ref_name }}' \
80+
--no-progress \
81+
--format markdown \
82+
>> result.text
83+
84+
# Read the result to the output of the step
85+
echo 'result<<EOF' >> "${GITHUB_OUTPUT}"
86+
cat result.text >> "${GITHUB_OUTPUT}"
87+
echo 'EOF' >> "${GITHUB_OUTPUT}"
88+
89+
- name: Compare branch '${{ github.head_ref || github.ref_name }}' against thresholds
90+
id: compare-benchmark
91+
run: |
92+
set -eu
93+
94+
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
95+
ENCODED_TIMESTAMP=$(date -u +"%Y-%m-%dT%H%%3A%M%%3A%SZ")
96+
TIMESTAMP_LINK="https://www.timeanddate.com/worldclock/fixedtime.html?iso=$ENCODED_TIMESTAMP"
97+
echo "## Benchmark check running at [$TIMESTAMP]($TIMESTAMP_LINK)" >> summary.text
98+
99+
# Disable 'set -e' to prevent the script from exiting on non-zero exit codes
100+
set +e
101+
swift package -c release --disable-sandbox \
102+
--package-path Benchmarks \
103+
benchmark thresholds check \
104+
'${{ github.head_ref || github.ref_name }}' \
105+
--path "$PWD/Benchmarks/Thresholds/" \
106+
--no-progress \
107+
--format markdown \
108+
>> summary.text
109+
echo "exit-status=$?" >> "${GITHUB_OUTPUT}"
110+
set -e
111+
112+
echo 'summary<<EOF' >> "${GITHUB_OUTPUT}"
113+
cat summary.text >> "${GITHUB_OUTPUT}"
114+
echo 'EOF' >> "${GITHUB_OUTPUT}"
115+
116+
- name: Cache .build
117+
if: steps.restore-cache.outputs.cache-hit != 'true'
118+
uses: actions/cache/save@v4
119+
with:
120+
path: Benchmarks/.build
121+
key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.event.pull_request.base.sha || github.event.after }}"
122+
123+
- name: Construct comment
124+
run: |
125+
set -eu
126+
127+
EXIT_CODE='${{ steps.compare-benchmark.outputs.exit-status }}'
128+
129+
echo 'PR_COMMENT<<EOF' >> "${GITHUB_ENV}"
130+
131+
echo '## [Benchmark](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) Report' >> "${GITHUB_ENV}"
132+
133+
case "${EXIT_CODE}" in
134+
0)
135+
echo '**✅ Pull request has no significant performance differences ✅**' >> "${GITHUB_ENV}"
136+
;;
137+
1)
138+
echo '**❌ Pull request has significant performance differences 📊**' >> "${GITHUB_ENV}"
139+
;;
140+
2)
141+
echo '**❌ Pull request has significant performance regressions 📉**' >> "${GITHUB_ENV}"
142+
;;
143+
4)
144+
echo '**❌ Pull request has significant performance improvements 📈**' >> "${GITHUB_ENV}"
145+
;;
146+
*)
147+
echo '**❌ Benchmark comparison failed to complete properly with exit code $EXIT_CODE ❌**' >> "${GITHUB_ENV}"
148+
;;
149+
esac
150+
151+
echo '<details>' >> "${GITHUB_ENV}"
152+
echo ' <summary> Click to expand comparison result </summary>' >> "${GITHUB_ENV}"
153+
echo '' >> "${GITHUB_ENV}"
154+
echo '${{ steps.compare-benchmark.outputs.summary }}' >> "${GITHUB_ENV}"
155+
echo '' >> "${GITHUB_ENV}"
156+
echo '</details>' >> "${GITHUB_ENV}"
157+
158+
echo '' >> "${GITHUB_ENV}"
159+
160+
echo '<details>' >> "${GITHUB_ENV}"
161+
echo ' <summary> Click to expand benchmark result </summary>' >> "${GITHUB_ENV}"
162+
echo '' >> "${GITHUB_ENV}"
163+
echo '${{ steps.read-benchmark.outputs.result }}' >> "${GITHUB_ENV}"
164+
echo '' >> "${GITHUB_ENV}"
165+
echo '</details>' >> "${GITHUB_ENV}"
166+
167+
echo 'EOF' >> "${GITHUB_ENV}"
168+
169+
- name: Output the comment as job summary
170+
run: echo '${{ env.PR_COMMENT }}' >> "${GITHUB_STEP_SUMMARY}"
171+
172+
- name: Comment in PR
173+
if: startsWith(github.event_name, 'pull_request')
174+
uses: thollander/actions-comment-pull-request@v3
175+
with:
176+
message: ${{ env.PR_COMMENT }}
177+
comment-tag: benchmark-ci-comment
178+
179+
- name: Exit with correct status
180+
run: |
181+
EXIT_CODE='${{ steps.compare-benchmark.outputs.exit-status }}'
182+
echo "Previous exit code was: $EXIT_CODE"
183+
exit $EXIT_CODE

‎.sourcekit-lsp/config.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../.sourcekit-lsp/config.json

‎Benchmarks/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../.gitignore

‎Benchmarks/.sourcekit-lsp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../.sourcekit-lsp

‎Benchmarks/.swift-format‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../.swift-format

‎Benchmarks/.vscode/launch.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"configurations": [
3+
{
4+
"type": "swift-lldb",
5+
"request": "launch",
6+
"args": [],
7+
"cwd": "${workspaceFolder:Benchmarks}",
8+
"name": "Debug Serializer",
9+
"program": "${workspaceFolder:Benchmarks}/.build/debug/Serializer",
10+
"preLaunchTask": "swift: Build Debug Serializer"
11+
},
12+
{
13+
"type": "swift-lldb",
14+
"request": "launch",
15+
"args": [],
16+
"cwd": "${workspaceFolder:Benchmarks}",
17+
"name": "Release Serializer",
18+
"program": "${workspaceFolder:Benchmarks}/.build/release/Serializer",
19+
"preLaunchTask": "swift: Build Release Serializer"
20+
},
21+
{
22+
"type": "swift-lldb",
23+
"request": "launch",
24+
"args": [],
25+
"cwd": "${workspaceFolder:Benchmarks}",
26+
"name": "Debug Parser",
27+
"program": "${workspaceFolder:Benchmarks}/.build/debug/Parser",
28+
"preLaunchTask": "swift: Build Debug Parser"
29+
},
30+
{
31+
"type": "swift-lldb",
32+
"request": "launch",
33+
"args": [],
34+
"cwd": "${workspaceFolder:Benchmarks}",
35+
"name": "Release Parser",
36+
"program": "${workspaceFolder:Benchmarks}/.build/release/Parser",
37+
"preLaunchTask": "swift: Build Release Parser"
38+
}
39+
]
40+
}

‎Benchmarks/Package.swift‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// swift-tools-version:6.0
2+
3+
import PackageDescription
4+
5+
let package = Package(
6+
name: "Benchmarks",
7+
platforms: [
8+
.macOS(.v13)
9+
],
10+
dependencies: [
11+
.package(path: "../"),
12+
.package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.27.0"),
13+
],
14+
targets: [
15+
.executableTarget(
16+
name: "Serializer",
17+
dependencies: [
18+
.product(name: "MultipartKit", package: "multipart-kit"),
19+
.product(name: "Benchmark", package: "package-benchmark"),
20+
],
21+
path: "Serializer",
22+
plugins: [
23+
.plugin(name: "BenchmarkPlugin", package: "package-benchmark")
24+
]
25+
),
26+
.executableTarget(
27+
name: "Parser",
28+
dependencies: [
29+
.product(name: "MultipartKit", package: "multipart-kit"),
30+
.product(name: "Benchmark", package: "package-benchmark"),
31+
],
32+
path: "Parser",
33+
plugins: [
34+
.plugin(name: "BenchmarkPlugin", package: "package-benchmark")
35+
]
36+
),
37+
]
38+
)

‎Benchmarks/Parser/AsyncSyncSequence.swift‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
extension Sequence {
2+
var async: AsyncSyncSequence<Self> {
3+
AsyncSyncSequence(self)
4+
}
5+
}
6+
7+
/// An asynchronous sequence composed from a synchronous sequence, that releases the reference to
8+
/// the base sequence when an iterator is created. So you can only iterate once.
9+
///
10+
/// Not safe. Only for testing purposes.
11+
/// Use `swift-algorithms`'s `AsyncSyncSequence`` instead if you're looking for something like this.
12+
final class AsyncSyncSequence<Base: Sequence>: AsyncSequence {
13+
typealias Element = Base.Element
14+
15+
struct Iterator: AsyncIteratorProtocol {
16+
var iterator: Base.Iterator?
17+
18+
init(_ iterator: Base.Iterator) {
19+
self.iterator = iterator
20+
}
21+
22+
mutating func next() async -> Base.Element? {
23+
iterator?.next()
24+
}
25+
}
26+
27+
private var base: Base?
28+
29+
init(_ base: Base) {
30+
self.base = base
31+
}
32+
33+
func makeAsyncIterator() -> Iterator {
34+
defer { self.base = nil } // release the reference so no CoW is triggered
35+
return Iterator(base.unsafelyUnwrapped.makeIterator())
36+
}
37+
}

‎Benchmarks/Parser/NoOpAsyncSequence.swift‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
struct NoOpAsyncSequence: AsyncSequence {
2+
typealias Element = ArraySlice<UInt8>
3+
4+
struct Iterator: AsyncIteratorProtocol {
5+
mutating func next() async -> Element? {
6+
nil
7+
}
8+
}
9+
10+
func makeAsyncIterator() -> Iterator {
11+
return Iterator()
12+
}
13+
}

0 commit comments

Comments
 (0)