Skip to content

Commit f130dbc

Browse files
alexeyralexeyr
committed
Initial benchmark version
1 parent 847edcd commit f130dbc

File tree

2 files changed

+180
-0
lines changed

2 files changed

+180
-0
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ yalc.lock
6767
/spec/dummy/.bsb.lock
6868
/spec/dummy/**/*.res.js
6969

70+
# Performance test results
71+
/bench_results
72+
7073
# Generated by ROR FS-based Registry
7174
generated
7275

‎spec/performance/bench.sh‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
#set -x # Uncomment for debugging commands
4+
5+
# Benchmark parameters
6+
TARGET="http://${BASE_URL:-localhost:3001}/${ROUTE:-server_side_hello_world_hooks}"
7+
# requests per second; if "max" will get maximum number of queries instead of a fixed rate
8+
RATE=${RATE:-50}
9+
# virtual users for k6
10+
VUS=${VUS:-100}
11+
DURATION_SEC=${DURATION_SEC:-10}
12+
DURATION="${DURATION_SEC}s"
13+
# Tools to run (comma-separated)
14+
TOOLS=${TOOLS:-fortio,vegeta,k6}
15+
16+
# Validate input parameters
17+
if ! { [ "$RATE" = "max" ] || { [[ "$RATE" =~ ^[0-9]+(\.[0-9]+)?$ ]] && (( $(bc -l <<< "$RATE > 0") )); }; }; then
18+
echo "Error: RATE must be 'max' or a positive number (got: '$RATE')" >&2
19+
exit 1
20+
fi
21+
if ! { [[ "$VUS" =~ ^[0-9]+$ ]] && [ "$VUS" -gt 0 ]; }; then
22+
echo "Error: VUS must be a positive integer (got: '$VUS')" >&2
23+
exit 1
24+
fi
25+
if ! { [[ "$DURATION_SEC" =~ ^[0-9]+(\.[0-9]+)?$ ]] && (( $(bc -l <<< "$DURATION_SEC > 0") )); }; then
26+
echo "Error: DURATION_SEC must be a positive number (got: '$DURATION_SEC')" >&2
27+
exit 1
28+
fi
29+
30+
OUTDIR="bench_results"
31+
32+
# Precompute checks for each tool
33+
RUN_FORTIO=0
34+
RUN_VEGETA=0
35+
RUN_K6=0
36+
[[ ",$TOOLS," == *",fortio,"* ]] && RUN_FORTIO=1
37+
[[ ",$TOOLS," == *",vegeta,"* ]] && RUN_VEGETA=1
38+
[[ ",$TOOLS," == *",k6,"* ]] && RUN_K6=1
39+
40+
for cmd in ${TOOLS//,/ } jq column awk tee; do
41+
if ! command -v "$cmd" >/dev/null 2>&1; then
42+
echo "Error: required tool '$cmd' is not installed" >&2
43+
exit 1
44+
fi
45+
done
46+
47+
TIMEOUT_SEC=60
48+
START=$(date +%s)
49+
until curl -fsS "$TARGET" >/dev/null; do
50+
if (( $(date +%s) - START > TIMEOUT_SEC )); then
51+
echo "Error: Target $TARGET not responding within ${TIMEOUT_SEC}s" >&2
52+
exit 1
53+
fi
54+
sleep 1
55+
done
56+
57+
mkdir -p "$OUTDIR"
58+
59+
if [ "$RATE" = "max" ]; then
60+
FORTIO_ARGS=(-qps 0)
61+
VEGETA_ARGS=(-rate=infinity)
62+
K6_SCENARIOS="{
63+
max_rate: {
64+
executor: 'shared-iterations',
65+
vus: $VUS,
66+
iterations: $((VUS * DURATION_SEC * 10)),
67+
maxDuration: '$DURATION'
68+
}
69+
}"
70+
else
71+
FORTIO_ARGS=(-qps "$RATE" -uniform)
72+
VEGETA_ARGS=(-rate="$RATE")
73+
K6_SCENARIOS="{
74+
constant_rate: {
75+
executor: 'constant-arrival-rate',
76+
rate: $RATE,
77+
timeUnit: '1s',
78+
duration: '$DURATION',
79+
preAllocatedVUs: $VUS,
80+
maxVUs: $((VUS * 10))
81+
}
82+
}"
83+
fi
84+
85+
if (( RUN_FORTIO )); then
86+
echo "===> Fortio"
87+
# TODO https://github.com/fortio/fortio/wiki/FAQ#i-want-to-get-the-best-results-what-flags-should-i-pass
88+
fortio load "${FORTIO_ARGS[@]}" -t "$DURATION" -timeout 30s -json "$OUTDIR/fortio.json" "$TARGET" \
89+
| tee "$OUTDIR/fortio.txt"
90+
fi
91+
92+
if (( RUN_VEGETA )); then
93+
echo
94+
echo "===> Vegeta"
95+
echo "GET $TARGET" | vegeta attack "${VEGETA_ARGS[@]}" -duration="$DURATION" \
96+
| tee "$OUTDIR/vegeta.bin" \
97+
| vegeta report | tee "$OUTDIR/vegeta.txt"
98+
vegeta report -type=json "$OUTDIR/vegeta.bin" > "$OUTDIR/vegeta.json"
99+
fi
100+
101+
if (( RUN_K6 )); then
102+
echo
103+
echo "===> k6"
104+
cat <<EOF > "$OUTDIR/k6_test.js"
105+
import http from 'k6/http';
106+
import { check } from 'k6';
107+
108+
export const options = {
109+
scenarios: $K6_SCENARIOS,
110+
};
111+
112+
export default function () {
113+
const response = http.get('$TARGET');
114+
check(response, {
115+
'status=200': r => r.status === 200,
116+
// you can add more if needed:
117+
// 'status=500': r => r.status === 500,
118+
});
119+
}
120+
EOF
121+
122+
k6 run --summary-export="$OUTDIR/k6_summary.json" --summary-trend-stats "min,avg,med,max,p(90),p(99)" "$OUTDIR/k6_test.js" | tee "$OUTDIR/k6.txt"
123+
fi
124+
125+
echo
126+
echo "===> Parsing results and generating summary"
127+
128+
echo -e "Tool\tRPS\tp50(ms)\tp90(ms)\tp99(ms)\tStatus" > "$OUTDIR/summary.txt"
129+
130+
if (( RUN_FORTIO )); then
131+
FORTIO_RPS=$(jq '.ActualQPS' "$OUTDIR/fortio.json" | awk '{printf "%.2f", $1}')
132+
FORTIO_P50=$(jq '.DurationHistogram.Percentiles[] | select(.Percentile==50) | .Value * 1000' "$OUTDIR/fortio.json" | awk '{printf "%.2f", $1}')
133+
FORTIO_P90=$(jq '.DurationHistogram.Percentiles[] | select(.Percentile==90) | .Value * 1000' "$OUTDIR/fortio.json" | awk '{printf "%.2f", $1}')
134+
FORTIO_P99=$(jq '.DurationHistogram.Percentiles[] | select(.Percentile==99) | .Value * 1000' "$OUTDIR/fortio.json" | awk '{printf "%.2f", $1}')
135+
FORTIO_STATUS=$(jq -r '.RetCodes | to_entries | map("\(.key)=\(.value)") | join(",")' "$OUTDIR/fortio.json")
136+
echo -e "Fortio\t$FORTIO_RPS\t$FORTIO_P50\t$FORTIO_P90\t$FORTIO_P99\t$FORTIO_STATUS" >> "$OUTDIR/summary.txt"
137+
fi
138+
139+
if (( RUN_VEGETA )); then
140+
# .throughput is successful_reqs/total_period, .rate is all_requests/attack_period
141+
VEGETA_RPS=$(jq '.throughput' "$OUTDIR/vegeta.json" | awk '{printf "%.2f", $1}')
142+
VEGETA_P50=$(jq '.latencies["50th"] / 1000000' "$OUTDIR/vegeta.json" | awk '{printf "%.2f", $1}')
143+
VEGETA_P90=$(jq '.latencies["90th"] / 1000000' "$OUTDIR/vegeta.json" | awk '{printf "%.2f", $1}')
144+
VEGETA_P99=$(jq '.latencies["99th"] / 1000000' "$OUTDIR/vegeta.json" | awk '{printf "%.2f", $1}')
145+
VEGETA_STATUS=$(jq -r '.status_codes | to_entries | map("\(.key)=\(.value)") | join(",")' "$OUTDIR/vegeta.json")
146+
echo -e "Vegeta\t$VEGETA_RPS\t$VEGETA_P50\t$VEGETA_P90\t$VEGETA_P99\t$VEGETA_STATUS" >> "$OUTDIR/summary.txt"
147+
fi
148+
149+
if (( RUN_K6 )); then
150+
K6_RPS=$(jq '.metrics.iterations.rate' "$OUTDIR/k6_summary.json" | awk '{printf "%.2f", $1}')
151+
K6_P50=$(jq '.metrics.http_req_duration.med' "$OUTDIR/k6_summary.json" | awk '{printf "%.2f", $1}')
152+
K6_P90=$(jq '.metrics.http_req_duration["p(90)"]' "$OUTDIR/k6_summary.json" | awk '{printf "%.2f", $1}')
153+
K6_P99=$(jq '.metrics.http_req_duration["p(99)"]' "$OUTDIR/k6_summary.json" | awk '{printf "%.2f", $1}')
154+
# Status: compute successful vs failed requests
155+
K6_REQS_TOTAL=$(jq '.metrics.http_reqs.count' "$OUTDIR/k6_summary.json")
156+
K6_STATUS=$(jq -r '
157+
.root_group.checks
158+
| to_entries
159+
| map(.key[7:] + "=" + (.value.passes|tostring))
160+
| join(",")
161+
' "$OUTDIR/k6_summary.json")
162+
K6_REQS_KNOWN_STATUS=$(jq -r '
163+
.root_group.checks
164+
| to_entries
165+
| map(.value.passes)
166+
| add
167+
' "$OUTDIR/k6_summary.json")
168+
K6_REQS_OTHER=$(( K6_REQS_TOTAL - K6_REQS_KNOWN_STATUS ))
169+
if [ "$K6_REQS_OTHER" -gt 0 ]; then
170+
K6_STATUS="$K6_STATUS,other=$K6_REQS_OTHER"
171+
fi
172+
echo -e "k6\t$K6_RPS\t$K6_P50\t$K6_P90\t$K6_P99\t$K6_STATUS" >> "$OUTDIR/summary.txt"
173+
fi
174+
175+
echo
176+
echo "Summary saved to $OUTDIR/summary.txt"
177+
column -t -s $'\t' "$OUTDIR/summary.txt"

0 commit comments

Comments
 (0)