Updated results summary

GATEOverflow · Oct 16, 2024 · c77698c · c77698c
1 parent 000872a
commit c77698c
Show file tree

Hide file tree

Showing 2 changed files with 107 additions and 33 deletions.
diff --git a/README.md b/README.md
@@ -1,19 +1,19 @@
 ```
-[2024-10-12 06:45:29,981 submission_checker1.py:2936 INFO] Results=21, NoResults=0, Power Results=0
-[2024-10-12 06:45:29,981 submission_checker1.py:2943 INFO] ---
-[2024-10-12 06:45:29,981 submission_checker1.py:2944 INFO] Closed Results=0, Closed Power Results=0
+[2024-10-16 10:38:18,552 submission_checker1.py:2936 INFO] Results=23, NoResults=0, Power Results=0
+[2024-10-16 10:38:18,552 submission_checker1.py:2943 INFO] ---
+[2024-10-16 10:38:18,552 submission_checker1.py:2944 INFO] Closed Results=0, Closed Power Results=0
 
-[2024-10-12 06:45:29,981 submission_checker1.py:2949 INFO] Open Results=21, Open Power Results=0
+[2024-10-16 10:38:18,552 submission_checker1.py:2949 INFO] Open Results=23, Open Power Results=0
 
-[2024-10-12 06:45:29,981 submission_checker1.py:2954 INFO] Network Results=0, Network Power Results=0
+[2024-10-16 10:38:18,552 submission_checker1.py:2954 INFO] Network Results=0, Network Power Results=0
 
-[2024-10-12 06:45:29,981 submission_checker1.py:2959 INFO] ---
-[2024-10-12 06:45:29,981 submission_checker1.py:2961 INFO] Systems=20, Power Systems=0
-[2024-10-12 06:45:29,981 submission_checker1.py:2962 INFO] Closed Systems=0, Closed Power Systems=0
-[2024-10-12 06:45:29,981 submission_checker1.py:2967 INFO] Open Systems=20, Open Power Systems=0
-[2024-10-12 06:45:29,981 submission_checker1.py:2972 INFO] Network Systems=0, Network Power Systems=0
-[2024-10-12 06:45:29,981 submission_checker1.py:2977 INFO] ---
-[2024-10-12 06:45:29,981 submission_checker1.py:2982 INFO] SUMMARY: submission looks OK
+[2024-10-16 10:38:18,552 submission_checker1.py:2959 INFO] ---
+[2024-10-16 10:38:18,552 submission_checker1.py:2961 INFO] Systems=22, Power Systems=0
+[2024-10-16 10:38:18,552 submission_checker1.py:2962 INFO] Closed Systems=0, Closed Power Systems=0
+[2024-10-16 10:38:18,552 submission_checker1.py:2967 INFO] Open Systems=22, Open Power Systems=0
+[2024-10-16 10:38:18,552 submission_checker1.py:2972 INFO] Network Systems=0, Network Power Systems=0
+[2024-10-16 10:38:18,552 submission_checker1.py:2977 INFO] ---
+[2024-10-16 10:38:18,552 submission_checker1.py:2982 INFO] SUMMARY: submission looks OK
 INFO:root:       ! call "postprocess" from /home/runner/CM/repos/mlcommons@cm4mlops/script/run-mlperf-inference-submission-checker/customize.py
 
 ```
@@ -40,4 +40,6 @@ INFO:root:       ! call "postprocess" from /home/runner/CM/repos/mlcommons@cm4ml
 | 17 | UNM-Roadrunners | available      | open       | datacenter   | d8727d7820f0 | d8727d7820f0-nvidia-gpu-TensorRT-scc24-base                | stable-diffusion-xl | stable-diffusion-xl | Offline    |  1.61345   | CLIP_SCORE: 15.495350807905197  FID_SCORE: 233.70007081366145 |                 1 | INTEL(R) XEON(R) PLATINUM 8580 |                          2 |                          60 | NVIDIA H100 NVL          |                       4 | open/UNM-Roadrunners/results/d8727d7820f0-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline          | TensorRT       | Ubuntu 20.04 (linux-5.14.0-427.37.1.el9_4.x86_64-glibc2.31) | Automated by MLCommons CM v2.4.0. |            1 |        0 | v4.1      |          0 | False       | Samples/s | int8                |
 | 18 | UNM-Roadrunners | available      | open       | datacenter   | 7b06989ff188 | 7b06989ff188-nvidia-gpu-TensorRT-scc24-base                | stable-diffusion-xl | stable-diffusion-xl | Offline    |  1.61853   | CLIP_SCORE: 15.527581945061684  FID_SCORE: 239.84603575864136 |                 1 | INTEL(R) XEON(R) PLATINUM 8580 |                          2 |                          60 | NVIDIA H100 NVL          |                       4 | open/UNM-Roadrunners/results/7b06989ff188-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline          | TensorRT       | Ubuntu 20.04 (linux-5.14.0-427.37.1.el9_4.x86_64-glibc2.31) | Automated by MLCommons CM v2.4.0. |            1 |        0 | v4.1      |          0 | False       | Samples/s | int8                |
 | 19 | UNM-Roadrunners | available      | open       | datacenter   | 4751fbf98487 | 4751fbf98487-reference-gpu-pytorch_v2.4.1-scc24-main       | stable-diffusion-xl | stable-diffusion-xl | Offline    |  0.665545  | CLIP_SCORE: 13.906862080842256  FID_SCORE: 84.21082273715388  |                 1 | INTEL(R) XEON(R) PLATINUM 8580 |                          2 |                          60 | NVIDIA H100 NVL          |                       1 | open/UNM-Roadrunners/results/4751fbf98487-reference-gpu-pytorch_v2.4.1-scc24-main/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-5.14.0-427.37.1.el9_4.x86_64-glibc2.35) | Automated by MLCommons CM v3.0.1. |            1 |        0 | v4.1      |          0 | False       | Samples/s | fp32                |
-| 20 | UNM-Roadrunners | available      | open       | datacenter   | roadrunner   | roadrunner-nvidia-gpu-TensorRT-scc24-base                  | stable-diffusion-xl | stable-diffusion-xl | Offline    |  1.60725   | CLIP_SCORE: 15.527581945061684  FID_SCORE: 239.84603550136694 |                 1 | INTEL(R) XEON(R) PLATINUM 8580 |                          2 |                          60 | NVIDIA H100 NVL          |                       4 | open/UNM-Roadrunners/results/roadrunner-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline            | TensorRT       | Ubuntu 20.04 (linux-5.14.0-427.37.1.el9_4.x86_64-glibc2.31) | Automated by MLCommons CM v2.4.0. |            1 |        0 | v4.1      |          0 | False       | Samples/s | int8                |
+| 20 | UNM-Roadrunners | available      | open       | datacenter   | roadrunner   | roadrunner-nvidia-gpu-TensorRT-scc24-base                  | stable-diffusion-xl | stable-diffusion-xl | Offline    |  1.60725   | CLIP_SCORE: 15.527581945061684  FID_SCORE: 239.84603550136694 |                 1 | INTEL(R) XEON(R) PLATINUM 8580 |                          2 |                          60 | NVIDIA H100 NVL          |                       4 | open/UNM-Roadrunners/results/roadrunner-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline            | TensorRT       | Ubuntu 20.04 (linux-5.14.0-427.37.1.el9_4.x86_64-glibc2.31) | Automated by MLCommons CM v2.4.0. |            1 |        0 | v4.1      |          0 | False       | Samples/s | int8                |
+| 21 | Dreambrook_Team | available      | open       | edge         | a65eca5827dc | a65eca5827dc-reference-cpu-pytorch_v2.4.1-default_config   | bert-99             | bert-99             | Offline    |  3.19276   | F1: 90.87487229720105                                         |                 1 | AMD EPYC-Milan Processor       |                         32 |                           1 | nan                      |                       0 | open/Dreambrook_Team/results/a65eca5827dc-reference-cpu-pytorch_v2.4.1-default_config/bert-99/offline         | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.1.112-1.el9.elrepo.x86_64-glibc2.35)  | Automated by MLCommons CM v3.2.1. |            1 |        0 | v4.1      |          0 | False       | Samples/s | fp32                |
+| 22 | Dreambrook_Team | available      | open       | edge         | 666dfb9811e4 | 666dfb9811e4-reference-gpu-pytorch-cu124                   | bert-99             | bert-99             | Offline    | 85.4467    | F1: 90.87601905331469                                         |                 1 | AMD EPYC-Milan Processor       |                         16 |                           1 | GRID A100X-20C           |                       1 | open/Dreambrook_Team/results/666dfb9811e4-reference-gpu-pytorch-cu124/bert-99/offline                         | pytorch        | Ubuntu 22.04 (linux-6.1.112-1.el9.elrepo.x86_64-glibc2.35)  | Automated by MLCommons CM v3.2.2. |            1 |        0 | v4.1      |          0 | False       | Samples/s | fp32                |
diff --git a/summary_results.json b/summary_results.json
@@ -1,6 +1,78 @@
 [
   {
     "ID": "4.1-0001",
+    "Submitter": "Dreambrook_Team",
+    "Availability": "available",
+    "Category": "open",
+    "Suite": "edge",
+    "System": "666dfb9811e4",
+    "Platform": "666dfb9811e4-reference-gpu-pytorch-cu124",
+    "UsedModel": "bert-99",
+    "Model": "bert-99",
+    "Scenario": "Offline",
+    "Accuracy": "F1: 90.87601905331469",
+    "Nodes": 1,
+    "Processor": "AMD EPYC-Milan Processor",
+    "host_processors_per_node": 16,
+    "host_processor_core_count": 1,
+    "Accelerator": "GRID A100X-20C",
+    "a#": 1,
+    "Location": "open/Dreambrook_Team/results/666dfb9811e4-reference-gpu-pytorch-cu124/bert-99/offline",
+    "Software": "pytorch",
+    "operating_system": "Ubuntu 22.04 (linux-6.1.112-1.el9.elrepo.x86_64-glibc2.35)",
+    "Notes": "Automated by MLCommons CM v3.2.2. ",
+    "compliance": 1,
+    "errors": 0,
+    "version": "v4.1",
+    "inferred": 0,
+    "has_power": false,
+    "weight_data_types": "fp32",
+    "p#": 16,
+    "Details": "https://github.com/mlcommons/submissions_inference_4.0/tree/main/open/Dreambrook_Team/results/666dfb9811e4-reference-gpu-pytorch-cu124",
+    "Code": "https://github.com/mlcommons/submissions_inference_4.0/tree/main/open/Dreambrook_Team/code",
+    "Unique ID (e.g. for Audit)": "edge/open/Dreambrook_Team/666dfb9811e4-reference-gpu-pytorch-cu124/bert-99",
+    "ColorKey": "availableDreambrook_Team",
+    "Performance_Result": 85.4467,
+    "Performance_Units": "Samples/s"
+  },
+  {
+    "ID": "4.1-0002",
+    "Submitter": "Dreambrook_Team",
+    "Availability": "available",
+    "Category": "open",
+    "Suite": "edge",
+    "System": "a65eca5827dc",
+    "Platform": "a65eca5827dc-reference-cpu-pytorch_v2.4.1-default_config",
+    "UsedModel": "bert-99",
+    "Model": "bert-99",
+    "Scenario": "Offline",
+    "Accuracy": "F1: 90.87487229720105",
+    "Nodes": 1,
+    "Processor": "AMD EPYC-Milan Processor",
+    "host_processors_per_node": 32,
+    "host_processor_core_count": 1,
+    "Accelerator": "",
+    "a#": "",
+    "Location": "open/Dreambrook_Team/results/a65eca5827dc-reference-cpu-pytorch_v2.4.1-default_config/bert-99/offline",
+    "Software": "pytorch v2.4.1",
+    "operating_system": "Ubuntu 22.04 (linux-6.1.112-1.el9.elrepo.x86_64-glibc2.35)",
+    "Notes": "Automated by MLCommons CM v3.2.1. ",
+    "compliance": 1,
+    "errors": 0,
+    "version": "v4.1",
+    "inferred": 0,
+    "has_power": false,
+    "weight_data_types": "fp32",
+    "p#": 32,
+    "Details": "https://github.com/mlcommons/submissions_inference_4.0/tree/main/open/Dreambrook_Team/results/a65eca5827dc-reference-cpu-pytorch_v2.4.1-default_config",
+    "Code": "https://github.com/mlcommons/submissions_inference_4.0/tree/main/open/Dreambrook_Team/code",
+    "Unique ID (e.g. for Audit)": "edge/open/Dreambrook_Team/a65eca5827dc-reference-cpu-pytorch_v2.4.1-default_config/bert-99",
+    "ColorKey": "availableDreambrook_Team",
+    "Performance_Result": 3.19276,
+    "Performance_Units": "Samples/s"
+  },
+  {
+    "ID": "4.1-0003",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -36,7 +108,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0002",
+    "ID": "4.1-0004",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -72,7 +144,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0003",
+    "ID": "4.1-0005",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -108,7 +180,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0004",
+    "ID": "4.1-0006",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -144,7 +216,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0005",
+    "ID": "4.1-0007",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -180,7 +252,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0006",
+    "ID": "4.1-0008",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -216,7 +288,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0007",
+    "ID": "4.1-0009",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -252,7 +324,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0008",
+    "ID": "4.1-0010",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -288,7 +360,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0009",
+    "ID": "4.1-0011",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -324,7 +396,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0010",
+    "ID": "4.1-0012",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -360,7 +432,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0011",
+    "ID": "4.1-0013",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -396,7 +468,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0012",
+    "ID": "4.1-0014",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -432,7 +504,7 @@
     "Performance_Units": "Tokens/s"
   },
   {
-    "ID": "4.1-0013",
+    "ID": "4.1-0015",
     "Submitter": "MLCommons",
     "Availability": "available",
     "Category": "open",
@@ -468,7 +540,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0014",
+    "ID": "4.1-0016",
     "Submitter": "UNM-Roadrunners",
     "Availability": "available",
     "Category": "open",
@@ -504,7 +576,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0015",
+    "ID": "4.1-0017",
     "Submitter": "UNM-Roadrunners",
     "Availability": "available",
     "Category": "open",
@@ -540,7 +612,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0016",
+    "ID": "4.1-0018",
     "Submitter": "UNM-Roadrunners",
     "Availability": "available",
     "Category": "open",
@@ -576,7 +648,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0017",
+    "ID": "4.1-0019",
     "Submitter": "UNM-Roadrunners",
     "Availability": "available",
     "Category": "open",
@@ -612,7 +684,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0018",
+    "ID": "4.1-0020",
     "Submitter": "UNM-Roadrunners",
     "Availability": "available",
     "Category": "open",
@@ -648,7 +720,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0019",
+    "ID": "4.1-0021",
     "Submitter": "UNM-Roadrunners",
     "Availability": "available",
     "Category": "open",
@@ -684,7 +756,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0020",
+    "ID": "4.1-0022",
     "Submitter": "UNM-Roadrunners",
     "Availability": "available",
     "Category": "open",
@@ -720,7 +792,7 @@
     "Performance_Units": "Samples/s"
   },
   {
-    "ID": "4.1-0021",
+    "ID": "4.1-0023",
     "Submitter": "UNM-Roadrunners",
     "Availability": "available",
     "Category": "open",