Skip to content

Commit 4f082fc

Browse files
committed
chore(ci): temporary hosted probe for tcp_reactor_perf stalls
Times every round of the tcp_reactor_perf loop on hosted macOS with main's build. Not for merging; the branch is deleted once the runs are read.
1 parent 9c6da14 commit 4f082fc

5 files changed

Lines changed: 390 additions & 0 deletions

File tree

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: Diagnostic tcp_reactor_perf stall probe
2+
3+
# Temporary and not for merging: times every round of the tcp_reactor_perf
4+
# loop on hosted macOS runners with main's build, and tells a round that
5+
# waited on the runtime from one where the host paused the whole program.
6+
7+
on:
8+
push:
9+
branches: ["diag/tcp-reactor-stall-probe"]
10+
11+
permissions:
12+
contents: read
13+
actions: read
14+
15+
jobs:
16+
probe:
17+
name: Stall probe ${{ matrix.part }}/3 (macOS arm64)
18+
runs-on: macos-latest
19+
timeout-minutes: 40
20+
strategy:
21+
fail-fast: false
22+
matrix:
23+
part: [1, 2, 3]
24+
steps:
25+
- uses: actions/checkout@v7
26+
27+
- name: Fetch main's build
28+
uses: actions/download-artifact@v8
29+
with:
30+
name: nera-build-macos
31+
path: ${{ runner.temp }}/compiler
32+
run-id: 43489667753
33+
github-token: ${{ github.token }}
34+
35+
- name: Build probes
36+
run: |
37+
set -euo pipefail
38+
tar -xf "$RUNNER_TEMP/compiler/nera-build.tar" -C "$GITHUB_WORKSPACE"
39+
sysctl -n hw.ncpu hw.memsize machdep.cpu.brand_string || true
40+
for m in hprobe hprobe_plain hprobe_inject; do
41+
./target/release/nera build "diag/$m.erl" -e "$m:test" -o "$RUNNER_TEMP/$m"
42+
done
43+
44+
- name: Probe without ticker
45+
run: python3 diag/run.py plain "$RUNNER_TEMP/hprobe_plain" 300 "$RUNNER_TEMP/probe.log"
46+
47+
- name: Probe with ticker
48+
run: python3 diag/run.py ticker "$RUNNER_TEMP/hprobe" 300 "$RUNNER_TEMP/probe.log"
49+
50+
- name: Controls
51+
run: |
52+
python3 diag/run.py control-delay "$RUNNER_TEMP/hprobe_inject" 5 "$RUNNER_TEMP/probe.log"
53+
python3 diag/run.py control-pause "$RUNNER_TEMP/hprobe" 5 "$RUNNER_TEMP/probe.log" 0.05
54+
55+
- name: Probe with ticker under CPU load
56+
run: |
57+
trap 'kill $(jobs -p) 2>/dev/null || true' EXIT
58+
for i in $(seq "$(sysctl -n hw.ncpu)"); do yes > /dev/null & done
59+
python3 diag/run.py ticker-loaded "$RUNNER_TEMP/hprobe" 100 "$RUNNER_TEMP/probe.log"
60+
61+
- name: Keep the per-run log
62+
if: always()
63+
uses: actions/upload-artifact@v7
64+
with:
65+
name: stall-probe-${{ matrix.part }}
66+
path: ${{ runner.temp }}/probe.log
67+
retention-days: 3

‎diag/hprobe.erl‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
-module(hprobe).
2+
-export([test/0]).
3+
4+
%% Per-round probe for the tcp_reactor_perf loop, as the test ran before it
5+
%% changed: five batches of 200 {active, once} rounds on one connection, then
6+
%% 200 warmup and 600 sustained rounds on a fresh one.
7+
%%
8+
%% A ticker process wakes every millisecond on the timer wheel, which does not
9+
%% go through the TCP reactor. For each round slower than 5 ms the probe prints
10+
%% the longest ticker gap that overlaps it. A slow round with a matching ticker
11+
%% gap means the whole program stopped (the host paused it); a slow round with
12+
%% no such gap means the round waited on the runtime alone.
13+
%%
14+
%% One line per run:
15+
%% old=ok|fail batches=[ms..] warm=ms sust=ms slow=[{Phase,Round,RoundUs,GapUs}] gaps=[GapUs >= 20 ms]
16+
test() ->
17+
Self = self(),
18+
Ticker = spawn(fun() -> tick(Self, erlang:monotonic_time(microsecond), []) end),
19+
A = pair(),
20+
Batches = [rounds(A, 200) || _ <- lists:seq(1, 5)],
21+
close(A),
22+
P = pair(),
23+
Warm = rounds(P, 200),
24+
Sust = rounds(P, 600),
25+
close(P),
26+
Ticker ! stop,
27+
Gaps = receive {gaps, G} -> G after 5000 -> [] end,
28+
BatchMs = [sum_us(B) div 1000 || B <- Batches],
29+
Sorted = lists:sort(BatchMs),
30+
Median = lists:nth(3, Sorted),
31+
WarmMs = sum_us(Warm) div 1000,
32+
SustMs = sum_us(Sust) div 1000,
33+
OldBatch = Median > 0 andalso lists:last(Sorted) > Median * 5,
34+
OldRatio = SustMs > max(1, WarmMs) * 9,
35+
Old = case OldBatch orelse OldRatio of true -> fail; false -> ok end,
36+
Phases = [{batch, R} || R <- Batches] ++ [{warm, Warm}, {sust, Sust}],
37+
Slow = lists:append([slow(Name, Rs, Gaps) || {Name, Rs} <- Phases]),
38+
Big = [U || {_, U} <- Gaps, U >= 20000],
39+
io:format("old=~p batches=~w warm=~p sust=~p slow=~w gaps=~w~n",
40+
[Old, BatchMs, WarmMs, SustMs, Slow, Big]),
41+
halt(0).
42+
43+
pair() ->
44+
{ok, LSock} = gen_tcp:listen(0, [{active, false}, binary, {packet, line}]),
45+
{ok, Port} = inet:port(LSock),
46+
{ok, CSock} = gen_tcp:connect({127,0,0,1}, Port, [{active, false}, binary]),
47+
{ok, SSock} = gen_tcp:accept(LSock),
48+
ok = gen_tcp:close(LSock),
49+
{SSock, CSock}.
50+
51+
close({S, C}) -> ok = gen_tcp:close(C), gen_tcp:close(S).
52+
53+
sum_us(Rounds) -> lists:sum([U || {_, U} <- Rounds]).
54+
55+
%% Rounds as {StartUs, DurationUs}.
56+
rounds(Pair, N) -> rounds(Pair, N, []).
57+
rounds(_Pair, 0, Acc) -> lists:reverse(Acc);
58+
rounds({S, C} = Pair, N, Acc) ->
59+
T0 = erlang:monotonic_time(microsecond),
60+
ok = inet:setopts(S, [{active, once}]),
61+
ok = gen_tcp:send(C, <<"P\r\n">>),
62+
receive {tcp, S, _} -> ok after 2000 -> erlang:error({timeout, N}) end,
63+
rounds(Pair, N - 1, [{T0, erlang:monotonic_time(microsecond) - T0} | Acc]).
64+
65+
slow(Name, Rounds, Gaps) ->
66+
Indexed = lists:zip(lists:seq(1, length(Rounds)), Rounds),
67+
[{Name, I, U, overlap(T0, T0 + U, Gaps)} || {I, {T0, U}} <- Indexed, U > 5000].
68+
69+
%% Longest ticker gap overlapping [From, To].
70+
overlap(From, To, Gaps) ->
71+
lists:max([0 | [G || {S, G} <- Gaps, S < To, S + G > From]]).
72+
73+
%% Ticker: records every gap over 3 ms between millisecond wakeups.
74+
tick(Parent, Last, Acc) ->
75+
receive
76+
stop -> Parent ! {gaps, lists:reverse(Acc)}
77+
after 1 ->
78+
Now = erlang:monotonic_time(microsecond),
79+
Acc1 = case Now - Last > 3000 of
80+
true -> [{Last, Now - Last} | Acc];
81+
false -> Acc
82+
end,
83+
tick(Parent, Now, Acc1)
84+
end.

‎diag/hprobe_inject.erl‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
-module(hprobe_inject).
2+
-export([test/0]).
3+
4+
%% Per-round probe for the tcp_reactor_perf loop, as the test ran before it
5+
%% changed: five batches of 200 {active, once} rounds on one connection, then
6+
%% 200 warmup and 600 sustained rounds on a fresh one.
7+
%%
8+
%% A ticker process wakes every millisecond on the timer wheel, which does not
9+
%% go through the TCP reactor. For each round slower than 5 ms the probe prints
10+
%% the longest ticker gap that overlaps it. A slow round with a matching ticker
11+
%% gap means the whole program stopped (the host paused it); a slow round with
12+
%% no such gap means the round waited on the runtime alone.
13+
%%
14+
%% One line per run:
15+
%% old=ok|fail batches=[ms..] warm=ms sust=ms slow=[{Phase,Round,RoundUs,GapUs}] gaps=[GapUs >= 20 ms]
16+
test() ->
17+
Self = self(),
18+
Ticker = spawn(fun() -> tick(Self, erlang:monotonic_time(microsecond), []) end),
19+
A = pair(),
20+
Batches = [rounds(A, 200) || _ <- lists:seq(1, 5)],
21+
close(A),
22+
P = pair(),
23+
Warm = rounds(P, 200),
24+
Sust = rounds(P, 600),
25+
close(P),
26+
Ticker ! stop,
27+
Gaps = receive {gaps, G} -> G after 5000 -> [] end,
28+
BatchMs = [sum_us(B) div 1000 || B <- Batches],
29+
Sorted = lists:sort(BatchMs),
30+
Median = lists:nth(3, Sorted),
31+
WarmMs = sum_us(Warm) div 1000,
32+
SustMs = sum_us(Sust) div 1000,
33+
OldBatch = Median > 0 andalso lists:last(Sorted) > Median * 5,
34+
OldRatio = SustMs > max(1, WarmMs) * 9,
35+
Old = case OldBatch orelse OldRatio of true -> fail; false -> ok end,
36+
Phases = [{batch, R} || R <- Batches] ++ [{warm, Warm}, {sust, Sust}],
37+
Slow = lists:append([slow(Name, Rs, Gaps) || {Name, Rs} <- Phases]),
38+
Big = [U || {_, U} <- Gaps, U >= 20000],
39+
io:format("old=~p batches=~w warm=~p sust=~p slow=~w gaps=~w~n",
40+
[Old, BatchMs, WarmMs, SustMs, Slow, Big]),
41+
halt(0).
42+
43+
pair() ->
44+
{ok, LSock} = gen_tcp:listen(0, [{active, false}, binary, {packet, line}]),
45+
{ok, Port} = inet:port(LSock),
46+
{ok, CSock} = gen_tcp:connect({127,0,0,1}, Port, [{active, false}, binary]),
47+
{ok, SSock} = gen_tcp:accept(LSock),
48+
ok = gen_tcp:close(LSock),
49+
{SSock, CSock}.
50+
51+
close({S, C}) -> ok = gen_tcp:close(C), gen_tcp:close(S).
52+
53+
sum_us(Rounds) -> lists:sum([U || {_, U} <- Rounds]).
54+
55+
%% Rounds as {StartUs, DurationUs}.
56+
rounds(Pair, N) -> rounds(Pair, N, []).
57+
rounds(_Pair, 0, Acc) -> lists:reverse(Acc);
58+
rounds({S, C} = Pair, N, Acc) ->
59+
T0 = erlang:monotonic_time(microsecond),
60+
ok = inet:setopts(S, [{active, once}]),
61+
ok = gen_tcp:send(C, <<"P\r\n">>),
62+
inject(),
63+
receive {tcp, S, _} -> ok after 2000 -> erlang:error({timeout, N}) end,
64+
rounds(Pair, N - 1, [{T0, erlang:monotonic_time(microsecond) - T0} | Acc]).
65+
66+
slow(Name, Rounds, Gaps) ->
67+
Indexed = lists:zip(lists:seq(1, length(Rounds)), Rounds),
68+
[{Name, I, U, overlap(T0, T0 + U, Gaps)} || {I, {T0, U}} <- Indexed, U > 5000].
69+
70+
%% Longest ticker gap overlapping [From, To].
71+
overlap(From, To, Gaps) ->
72+
lists:max([0 | [G || {S, G} <- Gaps, S < To, S + G > From]]).
73+
74+
%% Ticker: records every gap over 3 ms between millisecond wakeups.
75+
tick(Parent, Last, Acc) ->
76+
receive
77+
stop -> Parent ! {gaps, lists:reverse(Acc)}
78+
after 1 ->
79+
Now = erlang:monotonic_time(microsecond),
80+
Acc1 = case Now - Last > 3000 of
81+
true -> [{Last, Now - Last} | Acc];
82+
false -> Acc
83+
end,
84+
tick(Parent, Now, Acc1)
85+
end.
86+
87+
%% Control: one 100 ms delay at round 700 that is not a host pause.
88+
inject() ->
89+
G = case get(round) of undefined -> 1; X -> X + 1 end,
90+
put(round, G),
91+
case G of 700 -> timer:sleep(100); _ -> ok end.

‎diag/hprobe_plain.erl‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
-module(hprobe_plain).
2+
-export([test/0]).
3+
4+
%% Per-round probe for the tcp_reactor_perf loop, as the test ran before it
5+
%% changed: five batches of 200 {active, once} rounds on one connection, then
6+
%% 200 warmup and 600 sustained rounds on a fresh one.
7+
%%
8+
%% A ticker process wakes every millisecond on the timer wheel, which does not
9+
%% go through the TCP reactor. For each round slower than 5 ms the probe prints
10+
%% the longest ticker gap that overlaps it. A slow round with a matching ticker
11+
%% gap means the whole program stopped (the host paused it); a slow round with
12+
%% no such gap means the round waited on the runtime alone.
13+
%%
14+
%% One line per run:
15+
%% old=ok|fail batches=[ms..] warm=ms sust=ms slow=[{Phase,Round,RoundUs,GapUs}] gaps=[GapUs >= 20 ms]
16+
test() ->
17+
Self = self(),
18+
Ticker = spawn(fun() -> receive stop -> Self ! {gaps, []} end end),
19+
A = pair(),
20+
Batches = [rounds(A, 200) || _ <- lists:seq(1, 5)],
21+
close(A),
22+
P = pair(),
23+
Warm = rounds(P, 200),
24+
Sust = rounds(P, 600),
25+
close(P),
26+
Ticker ! stop,
27+
Gaps = receive {gaps, G} -> G after 5000 -> [] end,
28+
BatchMs = [sum_us(B) div 1000 || B <- Batches],
29+
Sorted = lists:sort(BatchMs),
30+
Median = lists:nth(3, Sorted),
31+
WarmMs = sum_us(Warm) div 1000,
32+
SustMs = sum_us(Sust) div 1000,
33+
OldBatch = Median > 0 andalso lists:last(Sorted) > Median * 5,
34+
OldRatio = SustMs > max(1, WarmMs) * 9,
35+
Old = case OldBatch orelse OldRatio of true -> fail; false -> ok end,
36+
Phases = [{batch, R} || R <- Batches] ++ [{warm, Warm}, {sust, Sust}],
37+
Slow = lists:append([slow(Name, Rs, Gaps) || {Name, Rs} <- Phases]),
38+
Big = [U || {_, U} <- Gaps, U >= 20000],
39+
io:format("old=~p batches=~w warm=~p sust=~p slow=~w gaps=~w~n",
40+
[Old, BatchMs, WarmMs, SustMs, Slow, Big]),
41+
halt(0).
42+
43+
pair() ->
44+
{ok, LSock} = gen_tcp:listen(0, [{active, false}, binary, {packet, line}]),
45+
{ok, Port} = inet:port(LSock),
46+
{ok, CSock} = gen_tcp:connect({127,0,0,1}, Port, [{active, false}, binary]),
47+
{ok, SSock} = gen_tcp:accept(LSock),
48+
ok = gen_tcp:close(LSock),
49+
{SSock, CSock}.
50+
51+
close({S, C}) -> ok = gen_tcp:close(C), gen_tcp:close(S).
52+
53+
sum_us(Rounds) -> lists:sum([U || {_, U} <- Rounds]).
54+
55+
%% Rounds as {StartUs, DurationUs}.
56+
rounds(Pair, N) -> rounds(Pair, N, []).
57+
rounds(_Pair, 0, Acc) -> lists:reverse(Acc);
58+
rounds({S, C} = Pair, N, Acc) ->
59+
T0 = erlang:monotonic_time(microsecond),
60+
ok = inet:setopts(S, [{active, once}]),
61+
ok = gen_tcp:send(C, <<"P\r\n">>),
62+
receive {tcp, S, _} -> ok after 2000 -> erlang:error({timeout, N}) end,
63+
rounds(Pair, N - 1, [{T0, erlang:monotonic_time(microsecond) - T0} | Acc]).
64+
65+
slow(Name, Rounds, Gaps) ->
66+
Indexed = lists:zip(lists:seq(1, length(Rounds)), Rounds),
67+
[{Name, I, U, overlap(T0, T0 + U, Gaps)} || {I, {T0, U}} <- Indexed, U > 5000].
68+
69+
%% Longest ticker gap overlapping [From, To].
70+
overlap(From, To, Gaps) ->
71+
lists:max([0 | [G || {S, G} <- Gaps, S < To, S + G > From]]).
72+
73+
%% Ticker: records every gap over 3 ms between millisecond wakeups.
74+
tick(Parent, Last, Acc) ->
75+
receive
76+
stop -> Parent ! {gaps, lists:reverse(Acc)}
77+
after 1 ->
78+
Now = erlang:monotonic_time(microsecond),
79+
Acc1 = case Now - Last > 3000 of
80+
true -> [{Last, Now - Last} | Acc];
81+
false -> Acc
82+
end,
83+
tick(Parent, Now, Acc1)
84+
end.

‎diag/run.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
"""Run the tcp_reactor_perf per-round probe many times and summarise it.
2+
3+
usage: run.py LABEL BINARY RUNS LOG [PAUSE_AT_SECONDS]
4+
Appends one line per run to LOG and prints a summary. With PAUSE_AT_SECONDS,
5+
the whole probe process is stopped for 100 ms that long after it starts, as
6+
a control for a host pause.
7+
"""
8+
import os
9+
import re
10+
import signal
11+
import statistics
12+
import subprocess
13+
import sys
14+
import time
15+
16+
label, binary, runs, log = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4]
17+
pause_at = float(sys.argv[5]) if len(sys.argv) > 5 else None
18+
slow_re = re.compile(r"\{(batch|warm|sust),(\d+),(\d+),(\d+)\}")
19+
stats_re = re.compile(r"events=(\d+) .*active_reads=(\d+)")
20+
21+
rc_bad = old_fail = 0
22+
buckets = {"5-20ms": 0, "20-90ms": 0, "90-130ms": 0, ">=130ms": 0}
23+
paused = [] # slow rounds >= 20 ms with a ticker gap covering half of them
24+
runtime = [] # slow rounds >= 20 ms without one
25+
ratios = []
26+
with open(log, "a") as out:
27+
for i in range(1, runs + 1):
28+
if pause_at is None:
29+
p = subprocess.run([binary], capture_output=True, text=True, timeout=120)
30+
else:
31+
proc = subprocess.Popen([binary], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
32+
time.sleep(pause_at)
33+
os.kill(proc.pid, signal.SIGSTOP)
34+
time.sleep(0.1)
35+
os.kill(proc.pid, signal.SIGCONT)
36+
o, e = proc.communicate(timeout=120)
37+
p = subprocess.CompletedProcess(proc.args, proc.returncode, o, e)
38+
line = p.stdout.strip().splitlines()[-1] if p.stdout.strip() else "(no output)"
39+
st = stats_re.search(p.stderr)
40+
out.write(f"{label} {i} rc={p.returncode} {line} {st.group(0) if st else ''}\n")
41+
if p.returncode != 0:
42+
rc_bad += 1
43+
if line.startswith("old=fail"):
44+
old_fail += 1
45+
if st and int(st.group(2)):
46+
ratios.append(int(st.group(1)) / int(st.group(2)))
47+
for phase, rnd, us, gap in slow_re.findall(line):
48+
us, gap = int(us), int(gap)
49+
ms = us / 1000
50+
key = "5-20ms" if ms < 20 else "20-90ms" if ms < 90 else "90-130ms" if ms < 130 else ">=130ms"
51+
buckets[key] += 1
52+
if ms >= 20:
53+
(paused if gap >= us / 2 else runtime).append((i, phase, int(rnd), round(ms, 1), round(gap / 1000, 1)))
54+
55+
print(f"== {label}: runs={runs} nonzero_exit={rc_bad} old_rule_fail={old_fail}")
56+
print(f" slow rounds by length: {buckets}")
57+
print(f" rounds >= 20 ms with a matching ticker gap (host pause): {len(paused)}")
58+
print(f" rounds >= 20 ms with no matching ticker gap (runtime only): {len(runtime)}")
59+
for r in runtime[:40]:
60+
print(f" runtime-only run={r[0]} {r[1]} round={r[2]} {r[3]} ms ticker_gap={r[4]} ms")
61+
for r in paused[:20]:
62+
print(f" host-pause run={r[0]} {r[1]} round={r[2]} {r[3]} ms ticker_gap={r[4]} ms")
63+
if ratios:
64+
print(f" reactor events per active read: min={min(ratios):.1f} median={statistics.median(ratios):.1f} max={max(ratios):.1f}")

0 commit comments

Comments
 (0)