Skip to content

Commit 22a65e9

Browse files
mlcommons-botmlcommons-bot
committed
Auto-merge updates from auto-update branch
2 parents 6424c8f + 05fab56 commit 22a65e9

File tree

1 file changed

+319
-0
lines changed

1 file changed

+319
-0
lines changed

‎add_results_summary.py

+319
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,319 @@
1+
import json
2+
import os
3+
import time
4+
import sys
5+
sys.path.insert(0, os.path.join("inference", "tools", "submission"))
6+
import submission_checker as checker # noqa
7+
8+
with open('summary_results.json') as f:
9+
data = json.load(f)
10+
#print(models_all)
11+
#print(platforms)
12+
13+
def get_header():
14+
css = """
15+
/* General table styles */
16+
table {
17+
width: 100%;
18+
border-collapse: collapse;
19+
margin: 20px 0;
20+
border: none;
21+
}
22+
23+
table.titlebarcontainer,.titlebar {
24+
text-align: center !important;
25+
}
26+
td.headerbar {
27+
text-align: center;
28+
font-size: x-large;
29+
}
30+
th, td {
31+
padding: 8px;
32+
text-align: left;
33+
vertical-align: top;
34+
}
35+
36+
/* Styling for the first table to make it stand out */
37+
table:first-of-type {
38+
background-color: #e8e8e8; /* Light grey background */
39+
color: #333; /* Darker text for contrast */
40+
font-weight: bold; /* Makes text bold */
41+
}
42+
43+
/* Styling inner tables to have borders */
44+
table table {
45+
border: 1px solid #ccc;
46+
}
47+
48+
/* More specific styling for headers and rows of inner tables */
49+
table table th, table table td {
50+
border: 1px solid #ccc;
51+
}
52+
53+
th {
54+
background-color: #f4f4f4;
55+
}
56+
57+
tr:nth-child(even) {
58+
background-color: #f9f9f9;
59+
}
60+
61+
tr:nth-child(odd) {
62+
background-color: #ffffff;
63+
}
64+
65+
/* Adjust column widths if necessary */
66+
th:nth-child(1), td:nth-child(1) { width: 20%; }
67+
th:nth-child(2), td:nth-child(2) { width: 30%; }
68+
"""
69+
html_header = f"""
70+
<head>
71+
<style type="text/css">
72+
{css}
73+
</style>
74+
</head>
75+
"""
76+
return html_header
77+
78+
def get_header_table(system_json):
79+
submitter = system_json.get('submitter')
80+
system_name = system_json.get('system_name')
81+
html = f"""
82+
<div class="resultpage">
83+
<div class="titlebarcontainer">
84+
<div class="logo">
85+
<a href="/" style="border: none"><img src="" alt="" /></a>
86+
</div>
87+
<div class="titlebar">
88+
<h1 class="title">MLPerf Inference v5.0</h1>
89+
<p style="font-size: smaller">Copyright 2019-2025 MLCommons</p>
90+
</div>
91+
</div>
92+
<table class="titlebarcontainer">
93+
<tr>
94+
<td class="headerbar" rowspan="2">
95+
<p>{submitter} </p>
96+
<p>{system_name} </p>
97+
</td>
98+
</tr>
99+
</table>
100+
<table class="datebar">
101+
<tbody>
102+
<tr>
103+
<th id="license_num"><a href="">MLPerf Inference Category:</a></th>
104+
<td id="license_num_val">Datacenter</td>
105+
<th id="test_date"><a href="">MLPerf Inference Division:</a></th>
106+
<td id="test_date_val">Closed</td>
107+
</tr>
108+
<tr>
109+
<th id="tester"><a href="">Submitted by:</a></th>
110+
<td id="tester_val">{submitter}</td>
111+
<th id="sw_avail"><a href="">Availability:</a></th>
112+
<td id="sw_avail_val">Available as of Feb 2025</td>
113+
</tr>
114+
</tbody>
115+
</table>
116+
"""
117+
return html
118+
119+
120+
def get_system_json(path):
121+
#import requests
122+
# Send a GET request to the URL
123+
#response = requests.get(url)
124+
125+
with open(path, "r") as f:
126+
data = json.load(f)
127+
# Check if the request was successful
128+
#if response.status_code == 200:
129+
# # Parse the JSON content
130+
# data = response.json()
131+
132+
return data
133+
134+
def get_accelerator_details_table(system_json):
135+
table = '<h3>Accelerator Details</h3><table>'
136+
for key,value in system_json.items():
137+
if not key.startswith("accelerator"):
138+
continue
139+
table += f"""<tr><td>{key}</td><td>{value}</td></tr>"""
140+
141+
table += "</table>"
142+
return table
143+
144+
def get_cpu_details_table(system_json):
145+
table = '<h3>Processor and Memory Details</h3><table>'
146+
hardware_fields = [ "processor", "cpu", "memory" ]
147+
for key,value in system_json.items():
148+
if any (a in key for a in hardware_fields) and "accelerator" not in key:
149+
table += f"""<tr><td>{key}</td><td>{value}</td></tr>"""
150+
151+
table += "</table>"
152+
return table
153+
def get_network_details_table(system_json):
154+
table = '<h3>Network and Interconnect Details</h3><table>'
155+
hardware_fields = [ "network", "nics" ]
156+
for key,value in system_json.items():
157+
if any (a in key for a in hardware_fields) and "accelerator" not in key:
158+
table += f"""<tr><td>{key}</td><td>{value}</td></tr>"""
159+
160+
table += "</table>"
161+
return table
162+
def get_hardware_details_table(system_json):
163+
table = '<h3>Other Hardware Details</h3><table>'
164+
hardware_fields = [ "hardware", "disk", "cooling", "power", "hw_" ]
165+
for key,value in system_json.items():
166+
if any (a in key for a in hardware_fields) and "accelerator" not in key:
167+
table += f"""<tr><td>{key}</td><td>{value}</td></tr>"""
168+
169+
table += "</table>"
170+
return table
171+
172+
def get_software_details_table(system_json):
173+
table = '<h3>Software Details</h3><table>'
174+
software_fields = [ "software", "framework", "firmware", "sw_", "operating_" ]
175+
for key,value in system_json.items():
176+
if any (a in key for a in software_fields):
177+
table += f"""<tr><td>{key}</td><td>{value}</td></tr>"""
178+
179+
table += "</table>"
180+
return table
181+
182+
# Initialize a dictionary to organize the data by 'Details'
183+
tables = {}
184+
185+
# Populate the dictionary with data
186+
for entry in data:
187+
details = entry['Details']
188+
if details not in tables:
189+
tables[details] = {}
190+
categories = [ "edge", "datacenter" ]
191+
for category in categories:
192+
if category not in entry['Suite']:
193+
continue
194+
if category not in tables[details]:
195+
tables[details][category] = {}
196+
if entry['Category'] not in tables[details][category]:
197+
tables[details][category][entry['Category']] = {}
198+
199+
if entry['Model'] not in tables[details][category][entry['Category']]:
200+
tables[details][category][entry['Category']] [entry['Model']] = {}
201+
if entry['Scenario'] not in tables[details][category][entry['Category']][entry['Model']]:
202+
tables[details][category][entry['Category']][entry['Model']][entry['Scenario']] = entry
203+
204+
# Now you can format each group in 'tables' as a markdown table
205+
for details, entries in tables.items():
206+
out = f"## {details}"
207+
208+
models_edge = [ "gptj-99", "gptj-99.9", "bert-99", "bert-99.9", "stable-diffusion-xl", "retinanet", "resnet", "3d-unet-99", "3d-unet-99.9" ]
209+
if "datacenter" in entries:
210+
models = [ "llama2-70b-99", "llama2-70b-99.9", "gptj-99", "gptj-99.9", "bert-99", "bert-99.9", "stable-diffusion-xl", "dlrm-v2-99", "dlrm-v2-99.9", "retinanet", "resnet", "3d-unet-99", "3d-unet-99.9" ]
211+
212+
html_table_head = f"""
213+
<h3>Results Table</h3>
214+
<table>
215+
<tr>
216+
<th rowspan="2">Model</th>
217+
<th rowspan="2">Accuracy Target</th>
218+
<th colspan="2">Server</th>
219+
<th colspan="2">Offline</th>
220+
</tr>
221+
<tr>
222+
<td>Metric</td>
223+
<td>Performance</td>
224+
<td>Metric</td>
225+
<td>Performance</td>
226+
</tr>
227+
"""
228+
for category in entries:
229+
for division, data in entries[category].items():
230+
html_table = html_table_head
231+
hardware_details = ''
232+
for model in models:
233+
if model in data:
234+
html_table += f"""<tr><td>{model}</td>"""
235+
if "closed" in division:
236+
version = data[model]["Offline"]["version"]
237+
acc_target = checker.MODEL_CONFIG[version]["accuracy-target"][model]
238+
i = 0
239+
acc_targets = []
240+
key = None
241+
for item in acc_target:
242+
if i%2 == 0:
243+
key = item
244+
else:
245+
acc_targets.append( (key, item))
246+
i+=1
247+
else:
248+
acc_targets = []
249+
acc_targets_list = []
250+
for item in acc_targets:
251+
acc_targets_list.append(f"""{item[0]}: {round(item[1], 4)}""")
252+
acc_targets_string = ", ".join(acc_targets_list)
253+
html_table += f"""<td>{acc_targets_string}</td>"""
254+
if "Server" in data[model]:
255+
html_table += f"""<td>{data[model]["Server"]["Performance_Units"]}</td> <td>{data[model]["Server"]["Performance_Result"]}</td>"""
256+
else:
257+
html_table += "<td></td><td></td>"
258+
if "Offline" in data[model]:
259+
details_split = details.split("/")
260+
details_split[9] = "systems"
261+
system = os.path.sep.join(details_split[7:11])
262+
#details_split[0] = "https://raw.githubusercontent.com"
263+
#system = details.replace("github.com", "raw.githubusercontent.com").replace("tree/", "refs/heads/").replace("results/", "systems/")
264+
system_json_path = f"""{system}.json"""
265+
system_json = get_system_json(system_json_path)
266+
header_table = get_header_table(system_json)
267+
accelerator_details = get_accelerator_details_table(system_json)
268+
cpu_details = get_cpu_details_table(system_json)
269+
hardware_details = get_hardware_details_table(system_json)
270+
software_details = get_software_details_table(system_json)
271+
network_details = get_network_details_table(system_json)
272+
html_table += f"""<td>{data[model]["Offline"]['Performance_Units']}</td> <td>{data[model]["Offline"]["Performance_Result"]}</td>"""
273+
else:
274+
html_table += "<td></td><td></td>"
275+
else:
276+
pass
277+
#html_table += "<td></td> <td></td>"
278+
#html_table += "<td></td> <td></td>"
279+
#html_table += "</tr>"
280+
html_table += "</table>"
281+
sut_name = os.path.basename(details)
282+
tmp_path = os.path.dirname(details)
283+
tmp_path = os.path.dirname(tmp_path)
284+
submitter = os.path.basename(tmp_path)
285+
out_path = os.path.join(division, submitter, "results", sut_name, "README.md")
286+
os.makedirs(os.path.dirname(out_path), exist_ok=True)
287+
288+
html_table = f"""
289+
290+
{header_table}
291+
<table>
292+
<tr><td>{accelerator_details}</td> <td>{cpu_details}</td> </tr>
293+
<tr><td >{hardware_details}</td> <td>{network_details}</td> </tr>
294+
<tr><td colspan="2">{software_details}</td> </tr>
295+
</table>
296+
{html_table}
297+
"""
298+
readme_content = f"""
299+
See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/blob/main/closed/{submitter}/results/{sut_name}/summary.html)
300+
{html_table}
301+
"""
302+
with open(out_path, "w") as f:
303+
f.write(readme_content)
304+
html_out_path = os.path.join(division, submitter, "results", sut_name, "summary.html")
305+
html_header = get_header()
306+
html = f"""
307+
<html>
308+
{html_header}
309+
<body>
310+
{html_table}
311+
</body>
312+
</html>
313+
"""
314+
with open(html_out_path, "w") as f:
315+
f.write(html)
316+
print(html_table)
317+
#sys.exit()
318+
319+

0 commit comments

Comments
 (0)