-
-
-
![]()
-
-
-
MLPerf Inference v5.0
-
Copyright 2019-2025 MLCommons
-
-
-
-
- """
- return html
-
-
-def get_system_json(path):
- #import requests
- # Send a GET request to the URL
- #response = requests.get(url)
-
- with open(path, "r") as f:
- data = json.load(f)
- # Check if the request was successful
- #if response.status_code == 200:
- # # Parse the JSON content
- # data = response.json()
-
- return data
-
-def get_accelerator_details_table(system_json):
- table = '
Accelerator Details
'
- for key,value in system_json.items():
- if not key.startswith("accelerator"):
- continue
- table += f"""| {key} | {value} |
"""
-
- table += "
"
- return table
-
-def get_cpu_details_table(system_json):
- table = '
Processor and Memory Details
'
- hardware_fields = [ "processor", "cpu", "memory" ]
- for key,value in system_json.items():
- if any (a in key for a in hardware_fields) and "accelerator" not in key:
- table += f"""| {key} | {value} |
"""
-
- table += "
"
- return table
-def get_network_details_table(system_json):
- table = '
Network and Interconnect Details
'
- hardware_fields = [ "network", "nics" ]
- for key,value in system_json.items():
- if any (a in key for a in hardware_fields) and "accelerator" not in key:
- table += f"""| {key} | {value} |
"""
-
- table += "
"
- return table
-def get_hardware_details_table(system_json):
- table = '
Other Hardware Details
'
- hardware_fields = [ "hardware", "disk", "cooling", "power", "hw_" ]
- for key,value in system_json.items():
- if any (a in key for a in hardware_fields) and "accelerator" not in key:
- table += f"""| {key} | {value} |
"""
-
- table += "
"
- return table
-
-def get_software_details_table(system_json):
- table = '
Software Details
'
- software_fields = [ "software", "framework", "firmware", "sw_", "operating_" ]
- for key,value in system_json.items():
- if any (a in key for a in software_fields):
- table += f"""| {key} | {value} |
"""
-
- table += "
"
- return table
-
-# Initialize a dictionary to organize the data by 'Details'
-tables = {}
-
-# Populate the dictionary with data
-for entry in data:
- details = entry['Details']
- if details not in tables:
- tables[details] = {}
- categories = [ "edge", "datacenter" ]
- for category in categories:
- if category not in entry['Suite']:
- continue
- if category not in tables[details]:
- tables[details][category] = {}
- if entry['Category'] not in tables[details][category]:
- tables[details][category][entry['Category']] = {}
-
- if entry['Model'] not in tables[details][category][entry['Category']]:
- tables[details][category][entry['Category']] [entry['Model']] = {}
- if entry['Scenario'] not in tables[details][category][entry['Category']][entry['Model']]:
- tables[details][category][entry['Category']][entry['Model']][entry['Scenario']] = entry
-
-# Now you can format each group in 'tables' as a markdown table
-for details, entries in tables.items():
- out = f"## {details}"
-
- models_edge = [ "gptj-99", "gptj-99.9", "bert-99", "bert-99.9", "stable-diffusion-xl", "retinanet", "resnet", "3d-unet-99", "3d-unet-99.9" ]
- if "datacenter" in entries:
- models = [ "llama2-70b-99", "llama2-70b-99.9", "gptj-99", "gptj-99.9", "bert-99", "bert-99.9", "stable-diffusion-xl", "dlrm-v2-99", "dlrm-v2-99.9", "retinanet", "resnet", "3d-unet-99", "3d-unet-99.9" ]
-
- html_table_head = f"""
-
Results Table
-
-
- | Model |
- Accuracy Target |
- Server |
- Offline |
-
-
- | Metric |
- Performance |
- Metric |
- Performance |
-
- """
- for category in entries:
- for division, data in entries[category].items():
- html_table = html_table_head
- hardware_details = ''
- for model in models:
- if model in data:
- html_table += f"""| {model} | """
- if "closed" in division:
- version = data[model]["Offline"]["version"]
- acc_target = checker.MODEL_CONFIG[version]["accuracy-target"][model]
- i = 0
- acc_targets = []
- key = None
- for item in acc_target:
- if i%2 == 0:
- key = item
- else:
- acc_targets.append( (key, item))
- i+=1
- else:
- acc_targets = []
- acc_targets_list = []
- for item in acc_targets:
- acc_targets_list.append(f"""{item[0]}: {round(item[1], 4)}""")
- acc_targets_string = ", ".join(acc_targets_list)
- html_table += f"""{acc_targets_string} | """
- if "Server" in data[model]:
- html_table += f"""{data[model]["Server"]["Performance_Units"]} | {data[model]["Server"]["Performance_Result"]} | """
- else:
- html_table += " | | "
- if "Offline" in data[model]:
- details_split = details.split("/")
- details_split[9] = "systems"
- system = os.path.sep.join(details_split[7:11])
- #details_split[0] = "https://raw.githubusercontent.com"
- #system = details.replace("github.com", "raw.githubusercontent.com").replace("tree/", "refs/heads/").replace("results/", "systems/")
- system_json_path = f"""{system}.json"""
- system_json = get_system_json(system_json_path)
- header_table = get_header_table(system_json)
- accelerator_details = get_accelerator_details_table(system_json)
- cpu_details = get_cpu_details_table(system_json)
- hardware_details = get_hardware_details_table(system_json)
- software_details = get_software_details_table(system_json)
- network_details = get_network_details_table(system_json)
- html_table += f"""{data[model]["Offline"]['Performance_Units']} | {data[model]["Offline"]["Performance_Result"]} | """
- else:
- html_table += " | | "
- else:
- pass
- #html_table += " | | "
- #html_table += " | | "
- #html_table += "
"
- html_table += "
"
- sut_name = os.path.basename(details)
- tmp_path = os.path.dirname(details)
- tmp_path = os.path.dirname(tmp_path)
- submitter = os.path.basename(tmp_path)
- out_path = os.path.join(division, submitter, "results", sut_name, "README.md")
- os.makedirs(os.path.dirname(out_path), exist_ok=True)
-
- html_table = f"""
-
-{header_table}
-
- | {accelerator_details} | {cpu_details} |
- | {hardware_details} | {network_details} |
- | {software_details} |
-
-{html_table}
-"""
- readme_content = f"""
-See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0/blob/main/closed/{submitter}/results/{sut_name}/summary.html)
-{html_table}
-"""
- with open(out_path, "w") as f:
- f.write(readme_content)
- html_out_path = os.path.join(division, submitter, "results", sut_name, "summary.html")
- html_header = get_header()
- html = f"""
-
-{html_header}
-
-{html_table}
-
-
- """
- with open(html_out_path, "w") as f:
- f.write(html)
- print(html_table)
- #sys.exit()
-
-