Skip to content

Commit d26ee78

Browse files
delthasdelthas
committed
Add throwaway lib-suite flakiness census workflow
Measures the lib functional-test failure rate of several trees under the real CI environment, N runs per arm, to separate the node-rdkafka downgrade from other causes. Not part of normal CI; poc/** is excluded from the tests workflow so this branch does not run it. Issue: BB-849
1 parent 52b9510 commit d26ee78

2 files changed

Lines changed: 196 additions & 0 deletions

File tree

‎.github/workflows/lib-census.yaml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
---
2+
# Throwaway workflow (BB-849): measures the lib-suite flakiness rate of several
3+
# trees under the real CI environment, N runs each. Not part of normal CI.
4+
name: lib-census
5+
6+
# workflow_dispatch only works from the default branch, so this runs on push
7+
# to the census branch instead; bump RUNS_PER_ARM and push to launch a round.
8+
on:
9+
push:
10+
branches:
11+
- poc/BB-849-lib-census
12+
13+
env:
14+
RUNS_PER_ARM: '1'
15+
16+
jobs:
17+
build:
18+
name: build ${{ matrix.name }}
19+
runs-on: ubuntu-latest
20+
permissions:
21+
contents: read
22+
packages: write
23+
strategy:
24+
matrix:
25+
include:
26+
- name: ci-kafka
27+
context: .github/dockerfiles/kafka
28+
file: ""
29+
- name: syntheticbucketd
30+
context: .
31+
file: .github/dockerfiles/syntheticbucketd/Dockerfile
32+
- name: ci-mongodb
33+
context: .github/dockerfiles/mongodb
34+
file: ""
35+
steps:
36+
- name: Checkout
37+
uses: actions/checkout@v4
38+
39+
- name: Set up Docker Buildx
40+
uses: docker/setup-buildx-action@v3
41+
42+
- name: Login to Registry
43+
uses: docker/login-action@v3
44+
with:
45+
registry: ghcr.io
46+
username: ${{ github.repository_owner }}
47+
password: ${{ github.token }}
48+
49+
- name: Build and push ${{ matrix.name }}
50+
uses: docker/build-push-action@v6
51+
with:
52+
push: true
53+
context: ${{ matrix.context }}
54+
file: ${{ matrix.file || format('{0}/Dockerfile', matrix.context) }}
55+
tags: ghcr.io/${{ github.repository }}/${{ matrix.name }}:${{ github.sha }}
56+
cache-from: type=gha,scope=${{ matrix.name }}
57+
cache-to: type=gha,mode=max,scope=${{ matrix.name }}
58+
59+
prepare:
60+
runs-on: ubuntu-latest
61+
outputs:
62+
matrix: ${{ steps.gen.outputs.matrix }}
63+
steps:
64+
- name: Build arm x run matrix
65+
id: gen
66+
run: |
67+
python3 - "$RUNS_PER_ARM" >> "$GITHUB_OUTPUT" <<'PY'
68+
import json
69+
import sys
70+
71+
n = int(sys.argv[1])
72+
# sha: tree to test. nrk: node-rdkafka version to force after install
73+
# ('' keeps whatever the tree's yarn.lock pins).
74+
arms = [
75+
{'arm': 'A-17jul-3.0.1', 'sha': 'c16e5ddb', 'nrk': ''},
76+
{'arm': 'B-16jul-3.6.1', 'sha': 'd7f3303d', 'nrk': ''},
77+
{'arm': 'C-head-3.0.1', 'sha': '52b9510f', 'nrk': ''},
78+
]
79+
include = [dict(a, idx=i) for a in arms for i in range(1, n + 1)]
80+
print('matrix=' + json.dumps(include))
81+
PY
82+
83+
census:
84+
needs: [build, prepare]
85+
name: ${{ matrix.arm }} ${{ matrix.idx }}
86+
runs-on: ubuntu-latest
87+
strategy:
88+
fail-fast: false
89+
max-parallel: 10
90+
matrix:
91+
include: ${{ fromJSON(needs.prepare.outputs.matrix) }}
92+
services:
93+
redis:
94+
image: redis:alpine
95+
ports:
96+
- 6379:6379
97+
syntheticbucketd:
98+
image: ghcr.io/${{ github.repository }}/syntheticbucketd:${{ github.sha }}
99+
ports:
100+
- 9001:9001
101+
zookeeper:
102+
image: zookeeper:3.9.4
103+
ports:
104+
- 2181:2181
105+
env:
106+
ZOOKEEPER_CLIENT_PORT: 2181
107+
ZOOKEEPER_TICK_TIME: 2000
108+
kafka:
109+
image: ghcr.io/${{ github.repository }}/ci-kafka:${{ github.sha }}
110+
credentials:
111+
username: ${{ github.repository_owner }}
112+
password: ${{ github.token }}
113+
ports:
114+
- 9092:9092
115+
env:
116+
KAFKA_BROKER_ID: 1
117+
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
118+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
119+
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
120+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
121+
mongo:
122+
image: ghcr.io/${{ github.repository}}/ci-mongodb:${{ github.sha }}
123+
ports:
124+
- 27017:27017
125+
- 27018:27018
126+
- 27019:27019
127+
steps:
128+
- name: Checkout
129+
uses: actions/checkout@v4
130+
with:
131+
fetch-depth: 0
132+
133+
- name: Check out arm tree ${{ matrix.sha }}
134+
run: git checkout --detach ${{ matrix.sha }}
135+
136+
- uses: actions/setup-node@v6
137+
with:
138+
node-version: '22'
139+
cache: yarn
140+
141+
- name: Install node dependencies
142+
run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
143+
144+
- name: Force node-rdkafka ${{ matrix.nrk }}
145+
if: matrix.nrk != ''
146+
run: yarn add node-rdkafka@${{ matrix.nrk }} --ignore-engines --network-concurrency 1
147+
148+
- name: Report effective versions
149+
run: |
150+
node -e "console.log('node-rdkafka', require('node-rdkafka/package.json').version,
151+
'/ librdkafka', require('node-rdkafka').librdkafkaVersion)"
152+
echo "tree: $(git rev-parse --short HEAD)"
153+
find tests/functional/lib -name '*.js' | sort
154+
155+
- name: Wait for services
156+
run: |
157+
timeout 60 bash -c 'until nc -z localhost 9092; do sleep 1; done'
158+
timeout 60 bash -c 'until nc -z localhost 2181; do sleep 1; done'
159+
timeout 60 bash -c 'until nc -z localhost 6379; do sleep 1; done'
160+
timeout 60 bash -c 'until nc -z localhost 27017; do sleep 1; done'
161+
162+
- name: Run lib tests
163+
env:
164+
BACKBEAT_CONFIG_FILE: tests/config.json
165+
run: .github/scripts/run_ft_tests.bash ft_test:lib 2>&1 | tee census.log
166+
167+
- name: Upload log
168+
if: always()
169+
uses: actions/upload-artifact@v4
170+
with:
171+
name: log-${{ matrix.arm }}-${{ matrix.idx }}
172+
path: census.log
173+
retention-days: 7
174+
175+
summary:
176+
needs: census
177+
if: always()
178+
runs-on: ubuntu-latest
179+
steps:
180+
- name: Tabulate per-arm outcomes
181+
env:
182+
GH_TOKEN: ${{ github.token }}
183+
run: |
184+
gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100" \
185+
--paginate --jq '.jobs[] | [.name, .conclusion] | @tsv' > jobs.tsv
186+
{
187+
echo "| arm | failed / total | rate |"
188+
echo "|---|---|---|"
189+
for arm in A-17jul-3.0.1 B-16jul-3.6.1 C-head-3.0.1; do
190+
t=$(grep -c "^$arm " jobs.tsv || true)
191+
f=$(awk -F'\t' -v a="$arm" '$1 ~ "^"a" " && $2=="failure"' jobs.tsv | wc -l)
192+
[ "$t" -gt 0 ] && echo "| $arm | $f / $t | $(( 100 * f / t ))% |"
193+
done
194+
} >> "$GITHUB_STEP_SUMMARY"

‎.github/workflows/tests.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ on:
88
push:
99
branches-ignore:
1010
- q/*/**
11+
# census-only: keep the normal suite off this throwaway branch
12+
- poc/**
1113

1214
jobs:
1315
build:

0 commit comments

Comments
 (0)