Skip to content

Commit 0961c8f

Browse files
author
Krzysztof Dziedzic
committed
test: set up itk nightly runs
1 parent 94ad594 commit 0961c8f

4 files changed

Lines changed: 294 additions & 78 deletions

File tree

‎.github/workflows/nightly.yaml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Nightly ITK
2+
3+
on:
4+
schedule:
5+
- cron: '0 2 * * *' # 2:00 AM UTC daily
6+
workflow_dispatch: # Allow manual execution
7+
8+
permissions:
9+
contents: write
10+
11+
jobs:
12+
nightly:
13+
name: Nightly ITK Run
14+
runs-on: ubuntu-latest
15+
16+
steps:
17+
- name: Checkout code
18+
uses: actions/checkout@v6
19+
20+
- name: Install uv
21+
uses: astral-sh/setup-uv@v7
22+
23+
- name: Run Nightly ITK Tests
24+
run: bash run_itk.sh
25+
working-directory: itk
26+
env:
27+
A2A_SAMPLES_REVISION: itk-v.021-alpha
28+
ITK_NIGHTLY_RUN: "True"
29+
30+
- name: Upload Results to Rolling Release
31+
uses: softprops/action-gh-release@v2
32+
with:
33+
tag_name: "nightly-metrics"
34+
prerelease: true
35+
files: |
36+
itk/itk_python.json
37+
env:
38+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎itk/process_results.py‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#!/usr/bin/env python3
2+
"""ITK Compatibility Metrics Processor.
3+
4+
Compiles test outcomes from raw JSON results, retrieves and aggregates historical
5+
runs from GitHub Release assets, and outputs the updated historical metrics log.
6+
"""
7+
8+
import datetime
9+
import json
10+
import logging
11+
import os
12+
import pathlib
13+
import sys
14+
import urllib.error
15+
import urllib.request
16+
17+
18+
# --- CONSTANTS ---
19+
RESULTS_FILE = 'raw_results.json'
20+
HISTORY_OUTPUT_FILE = 'itk_python.json'
21+
HISTORY_URL = 'https://github.com/a2aproject/a2a-python/releases/download/nightly-metrics/itk_python.json'
22+
SCENARIOS_FILE = 'scenarios.json'
23+
DEFAULT_HISTORY_LIMIT = 50
24+
25+
HTTP_STATUS_OK = 200
26+
HTTP_STATUS_NOT_FOUND = 404
27+
28+
# Configure logging to match standard ITK formatting
29+
logging.basicConfig(
30+
level=logging.INFO,
31+
)
32+
logger = logging.getLogger(__name__)
33+
34+
35+
def load_raw_results(filepath: str) -> dict:
36+
"""Loads the raw compatibility results from raw_results.json."""
37+
path = pathlib.Path(filepath)
38+
if not path.exists():
39+
logger.error('Results file %s not found.', filepath)
40+
raise SystemExit(1)
41+
42+
try:
43+
with path.open() as f:
44+
return json.load(f)
45+
except (OSError, json.JSONDecodeError):
46+
logger.exception('Error loading results JSON')
47+
raise SystemExit(1) from None
48+
49+
50+
def fetch_existing_history(url: str) -> list:
51+
"""Fetches the existing compatibility history from the GitHub release asset.
52+
53+
If the asset does not exist (HTTP 404), a fresh empty history list is returned.
54+
For all other network or server errors, the script exits with a non-zero status
55+
to prevent overwriting and losing historical metrics.
56+
"""
57+
try:
58+
req = urllib.request.Request( # noqa: S310
59+
url, headers={'User-Agent': 'Mozilla/5.0'}
60+
)
61+
with urllib.request.urlopen(req, timeout=15) as response: # noqa: S310
62+
if response.status == HTTP_STATUS_OK:
63+
history = json.loads(response.read().decode('utf-8'))
64+
logger.info(
65+
'Successfully retrieved history. Current entries: %d',
66+
len(history),
67+
)
68+
return history
69+
logger.error(
70+
'Unexpected HTTP status when downloading existing history: %d',
71+
response.status,
72+
)
73+
raise SystemExit(1) # noqa: TRY301
74+
except urllib.error.HTTPError as e:
75+
if e.code == HTTP_STATUS_NOT_FOUND:
76+
logger.warning(
77+
'No existing history found (HTTP %d). Initializing fresh history.',
78+
e.code,
79+
)
80+
return []
81+
logger.exception(
82+
'HTTP error downloading existing history: %d. Aborting to preserve metrics.',
83+
e.code,
84+
)
85+
raise SystemExit(1) from None
86+
except Exception:
87+
logger.exception(
88+
'Failed to download existing history. Aborting to preserve metrics.'
89+
)
90+
raise SystemExit(1) from None
91+
92+
93+
def load_scenarios(filepath: str) -> list:
94+
"""Loads the list of tests from the scenarios.json definitions."""
95+
path = pathlib.Path(filepath)
96+
if not path.exists():
97+
logger.error('Scenarios file %s not found.', filepath)
98+
raise SystemExit(1)
99+
100+
try:
101+
with path.open() as f:
102+
data = json.load(f)
103+
return data['tests']
104+
except (OSError, json.JSONDecodeError, KeyError):
105+
logger.exception('Failed to load scenarios.json definitions')
106+
raise SystemExit(1) from None
107+
108+
109+
def save_history(filepath: str, history: list) -> None:
110+
"""Saves the updated history back to disk as a release asset candidate."""
111+
path = pathlib.Path(filepath)
112+
try:
113+
with path.open('w') as f:
114+
json.dump(history, f, indent=2)
115+
logger.info(
116+
'Successfully compiled and wrote nightly history to: %s',
117+
filepath,
118+
)
119+
except (OSError, TypeError):
120+
logger.exception('Error writing history file')
121+
sys.exit(1)
122+
123+
124+
def main() -> None:
125+
"""Orchestrates nightly ITK metrics processing and compiles rolling history."""
126+
# 1. Load raw compatibility results
127+
data = load_raw_results(RESULTS_FILE)
128+
all_passed = data.get('all_passed', False)
129+
results = data.get('results', {})
130+
131+
# 2. Fetch existing history from rolling release
132+
history = fetch_existing_history(HISTORY_URL)
133+
134+
# 3. Load scenarios list for metadata
135+
scenarios_list = load_scenarios(SCENARIOS_FILE)
136+
137+
# Merge definitions with current outcomes
138+
compiled_scenarios = []
139+
for scenario in scenarios_list:
140+
name = scenario.get('name')
141+
passed = results.get(name, False)
142+
combined = dict(scenario)
143+
combined['passed'] = passed
144+
compiled_scenarios.append(combined)
145+
146+
# 4. Compile new run metadata
147+
new_run = {
148+
'timestamp': datetime.datetime.now(datetime.timezone.utc).isoformat(),
149+
'commit_sha': os.environ.get('GITHUB_SHA', 'local-dev'),
150+
'github_run_id': os.environ.get('GITHUB_RUN_ID', '0'),
151+
'all_passed': all_passed,
152+
'scenarios': compiled_scenarios,
153+
}
154+
155+
# 5. Merge and Prune rolling window
156+
history.append(new_run)
157+
history_limit = int(
158+
os.environ.get('ITK_HISTORY_LIMIT', str(DEFAULT_HISTORY_LIMIT))
159+
)
160+
if len(history) > history_limit:
161+
history = history[-history_limit:]
162+
logger.info('Pruned history to last %d entries.', history_limit)
163+
164+
# 6. Save candidates back to disk
165+
save_history(HISTORY_OUTPUT_FILE, history)
166+
sys.exit(0)
167+
168+
169+
if __name__ == '__main__':
170+
main()

‎itk/run_itk.sh‎

Lines changed: 14 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -112,83 +112,18 @@ fi
112112
echo "ITK Service is up! Sending compatibility test request..."
113113
RESPONSE=$(curl -s -X POST http://127.0.0.1:8000/run \
114114
-H "Content-Type: application/json" \
115-
-d '{
116-
"tests": [
117-
{
118-
"name": "Star Topology (Full) - JSONRPC & GRPC",
119-
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
120-
"traversal": "euler",
121-
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
122-
"protocols": ["jsonrpc", "grpc"],
123-
"behavior": "send_message"
124-
},
125-
{
126-
"name": "Star Topology (No Go v03) - HTTP_JSON",
127-
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
128-
"traversal": "euler",
129-
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
130-
"protocols": ["http_json"],
131-
"behavior": "send_message"
132-
},
133-
{
134-
"name": "Star Topology (Full) - JSONRPC & GRPC (Streaming)",
135-
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
136-
"traversal": "euler",
137-
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
138-
"protocols": ["jsonrpc", "grpc"],
139-
"streaming": true,
140-
"behavior": "send_message"
141-
},
142-
{
143-
"name": "Star Topology (No Go v03) - HTTP_JSON (Streaming)",
144-
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
145-
"traversal": "euler",
146-
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
147-
"protocols": ["http_json"],
148-
"streaming": true,
149-
"behavior": "send_message"
150-
},
151-
{
152-
"name": "Push Notification Test - JSONRPC & GRPC",
153-
"sdks": ["current", "python_v10", "python_v03", "go_v03"],
154-
"traversal": "euler",
155-
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
156-
"protocols": ["jsonrpc", "grpc"],
157-
"behavior": "push_notification"
158-
},
159-
{
160-
"name": "Push Notification Test - HTTP_JSON",
161-
"sdks": ["current", "python_v10", "python_v03"],
162-
"traversal": "euler",
163-
"edges": ["0->1", "0->2", "1->0", "2->0"],
164-
"protocols": ["http_json"],
165-
"behavior": "push_notification"
166-
},
167-
{
168-
"name": "Resubscribe Test - JSONRPC",
169-
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
170-
"traversal": "euler",
171-
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
172-
"protocols": ["jsonrpc"],
173-
"streaming": true,
174-
"behavior": "resubscribe"
175-
},
176-
{
177-
"name": "Resubscribe Test - Python & Go Non-JSONRPC Protocols",
178-
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
179-
"traversal": "euler",
180-
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
181-
"protocols": ["grpc", "http_json"],
182-
"streaming": true,
183-
"behavior": "resubscribe"
184-
}
185-
]
186-
}')
187-
188-
echo "--------------------------------------------------------"
189-
echo "ITK TEST RESULTS:"
190-
echo "--------------------------------------------------------"
191-
echo "$RESPONSE" | python3 -c "
115+
-d @scenarios.json)
116+
117+
if [ "${ITK_NIGHTLY_RUN^^}" = "TRUE" ]; then
118+
echo "Nightly run detected. Saving raw results and running process_results.py..."
119+
echo "$RESPONSE" > raw_results.json
120+
python3 process_results.py
121+
RESULT=$?
122+
else
123+
echo "--------------------------------------------------------"
124+
echo "ITK TEST RESULTS:"
125+
echo "--------------------------------------------------------"
126+
echo "$RESPONSE" | python3 -c "
192127
import sys, json
193128
try:
194129
data = json.load(sys.stdin)
@@ -206,7 +141,8 @@ except Exception as e:
206141
print(f'Raw response: {data if \"data\" in locals() else \"no data\"}')
207142
sys.exit(1)
208143
"
209-
RESULT=$?
144+
RESULT=$?
145+
fi
210146
set -e
211147

212148
if [ $RESULT -ne 0 ]; then

‎itk/scenarios.json‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"tests": [
3+
{
4+
"name": "Star Topology (Full) - JSONRPC & GRPC",
5+
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
6+
"traversal": "euler",
7+
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
8+
"protocols": ["jsonrpc", "grpc"],
9+
"behavior": "send_message"
10+
},
11+
{
12+
"name": "Star Topology (No Go v03) - HTTP_JSON",
13+
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
14+
"traversal": "euler",
15+
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
16+
"protocols": ["http_json"],
17+
"behavior": "send_message"
18+
},
19+
{
20+
"name": "Star Topology (Full) - JSONRPC & GRPC (Streaming)",
21+
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
22+
"traversal": "euler",
23+
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
24+
"protocols": ["jsonrpc", "grpc"],
25+
"streaming": true,
26+
"behavior": "send_message"
27+
},
28+
{
29+
"name": "Star Topology (No Go v03) - HTTP_JSON (Streaming)",
30+
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
31+
"traversal": "euler",
32+
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
33+
"protocols": ["http_json"],
34+
"streaming": true,
35+
"behavior": "send_message"
36+
},
37+
{
38+
"name": "Push Notification Test - JSONRPC & GRPC",
39+
"sdks": ["current", "python_v10", "python_v03", "go_v03"],
40+
"traversal": "euler",
41+
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
42+
"protocols": ["jsonrpc", "grpc"],
43+
"behavior": "push_notification"
44+
},
45+
{
46+
"name": "Push Notification Test - HTTP_JSON",
47+
"sdks": ["current", "python_v10", "python_v03"],
48+
"traversal": "euler",
49+
"edges": ["0->1", "0->2", "1->0", "2->0"],
50+
"protocols": ["http_json"],
51+
"behavior": "push_notification"
52+
},
53+
{
54+
"name": "Resubscribe Test - JSONRPC",
55+
"sdks": ["current", "python_v10", "python_v03", "go_v10", "go_v03"],
56+
"traversal": "euler",
57+
"edges": ["0->1", "0->2", "0->3", "0->4", "1->0", "2->0", "3->0", "4->0"],
58+
"protocols": ["jsonrpc"],
59+
"streaming": true,
60+
"behavior": "resubscribe"
61+
},
62+
{
63+
"name": "Resubscribe Test - Python & Go Non-JSONRPC Protocols",
64+
"sdks": ["current", "python_v10", "python_v03", "go_v10"],
65+
"traversal": "euler",
66+
"edges": ["0->1", "0->2", "0->3", "1->0", "2->0", "3->0"],
67+
"protocols": ["grpc", "http_json"],
68+
"streaming": true,
69+
"behavior": "resubscribe"
70+
}
71+
]
72+
}

0 commit comments

Comments
 (0)