Results from self hosted Github actions - NVIDIARTX4090

GATEOverflow · Nov 7, 2024 · af71a75 · af71a75
1 parent 37784b6
commit af71a75
Show file tree

Hide file tree

Showing 13 changed files with 462 additions and 463 deletions.
diff --git a/.../measurements/arjun_spr-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/README.md b/.../measurements/arjun_spr-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/README.md
@@ -1,3 +1,3 @@
 | Model               | Scenario   | Accuracy             |   Throughput | Latency (in ms)   |
 |---------------------|------------|----------------------|--------------|-------------------|
-| stable-diffusion-xl | offline    | (16.3689, 237.82579) |        0.384 | -                 |
+| stable-diffusion-xl | offline    | (16.3689, 237.82579) |        0.383 | -                 |
diff --git a/...rence-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/README.md b/...rence-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/README.md
@@ -18,7 +18,7 @@ pip install -U cmind
 
 cm rm cache -f
 
-cm pull repo gateoverflow@cm4mlops --checkout=ea39555bb4231038de51e1e33812dee952b946aa
+cm pull repo gateoverflow@cm4mlops --checkout=58a2259330bf902d2b314d2d1bdaeb53706e9ed1
 
 cm run script \
 	--tags=app,mlperf,inference,generic,_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline \
@@ -44,7 +44,8 @@ cm run script \
 	--env.CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX4=scc24-base \
 	--env.CM_DOCKER_IMAGE_NAME=scc24-reference \
 	--env.CM_MLPERF_LOADGEN_ALL_MODES=yes \
-	--env.CM_MLPERF_LAST_RELEASE=v4.1 \
+	--env.CM_MLPERF_INFERENCE_SOURCE_VERSION=4.1.23 \
+	--env.CM_MLPERF_LAST_RELEASE=v4.0 \
 	--env.CM_TMP_CURRENT_PATH=/home/arjun/actions-runner/_work/cm4mlops/cm4mlops \
 	--env.CM_TMP_PIP_VERSION_STRING= \
 	--env.CM_MODEL=sdxl \
@@ -65,6 +66,7 @@ cm run script \
 	--add_deps_recursive.coco2014-preprocessed.tags=_size.50,_with-sample-ids \
 	--add_deps_recursive.coco2014-dataset.tags=_size.50,_with-sample-ids \
 	--add_deps_recursive.nvidia-preprocess-data.extra_cache_tags=scc24-base \
+	--add_deps_recursive.inference-src.tags=_branch.dev \
 	--v=False \
 	--print_env=False \
 	--print_deps=False \
@@ -94,4 +96,4 @@ Model Precision: fp32
 `FID_SCORE`: `237.82579`, Required accuracy for closed division `>= 23.01086` and `<= 23.95008`
 
 ### Performance Results 
-`Samples per second`: `0.383508`
+`Samples per second`: `0.383039`
diff --git a/...ence-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/cm-version-info.json b/...ence-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/cm-version-info.json
@@ -31,22 +31,62 @@
       }
     },
     {
-      "get,mlcommons,inference,src": {
+      "detect,os": {
+        "script_uid": "863735b7db8c44fc",
+        "script_alias": "detect-os",
+        "script_tags": "detect-os,detect,os,info",
+        "script_variations": "",
+        "version": "",
+        "parent": "get-mlperf-inference-src,4b57186581024797 ( branch.dev )"
+      }
+    },
+    {
+      "get,python3": {
+        "script_uid": "d0b5dd74373f4a62",
+        "script_alias": "get-python3",
+        "script_tags": "get,python,python3,get-python,get-python3",
+        "script_variations": "",
+        "version": "3.10.12",
+        "parent": "get-mlperf-inference-src,4b57186581024797 ( branch.dev )"
+      }
+    },
+    {
+      "detect,os": {
+        "script_uid": "863735b7db8c44fc",
+        "script_alias": "detect-os",
+        "script_tags": "detect-os,detect,os,info",
+        "script_variations": "",
+        "version": "",
+        "parent": "get-git-repo,ed603e7292974f10 ( branch.dev,_repo.https://github.com/mlcommons/inference )"
+      }
+    },
+    {
+      "get,git,repo,_branch.dev,_repo.https://github.com/mlcommons/inference": {
+        "script_uid": "ed603e7292974f10",
+        "script_alias": "get-git-repo",
+        "script_tags": "get,git,repo,repository,clone",
+        "script_variations": "branch.dev,repo.https://github.com/mlcommons/inference",
+        "version": "",
+        "parent": "get-mlperf-inference-src,4b57186581024797 ( branch.dev )"
+      }
+    },
+    {
+      "get,mlcommons,inference,src,_branch.dev": {
         "script_uid": "4b57186581024797",
         "script_alias": "get-mlperf-inference-src",
         "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
-        "script_variations": "",
-        "version": "master-git-c8c1e61f1a2164b5797504f1991d5c7aeea2364f",
+        "script_variations": "branch.dev",
+        "version": "custom-git-8d3b8ab09ac392b5a8656ad07d37fb8d7942595b",
         "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline )"
       }
     },
     {
-      "get,mlperf,inference,src": {
+      "get,mlperf,inference,src,_branch.dev": {
         "script_uid": "4b57186581024797",
         "script_alias": "get-mlperf-inference-src",
         "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
-        "script_variations": "",
-        "version": "master-git-c8c1e61f1a2164b5797504f1991d5c7aeea2364f",
+        "script_variations": "branch.dev",
+        "version": "custom-git-8d3b8ab09ac392b5a8656ad07d37fb8d7942595b",
         "parent": "get-mlperf-inference-utils,e341e5f86d8342e5"
       }
     },
@@ -227,7 +267,7 @@
         "script_tags": "detect-os,detect,os,info",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -247,7 +287,7 @@
         "script_tags": "detect,cpu,detect-cpu,info",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -257,7 +297,7 @@
         "script_tags": "get,sys-utils-cm",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -267,7 +307,7 @@
         "script_tags": "get,python,python3,get-python,get-python3",
         "script_variations": "",
         "version": "3.10.12",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -277,7 +317,7 @@
         "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d",
         "script_variations": "cudnn",
         "version": "12.4",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -287,7 +327,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "torch_cuda",
         "version": "2.5.1",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -297,7 +337,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "torchvision_cuda",
         "version": "0.20.1",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -307,7 +347,7 @@
         "script_tags": "get,raw,ml-model,stable-diffusion,sdxl,text-to-image",
         "script_variations": "pytorch,fp16",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -317,7 +357,7 @@
         "script_tags": "get,dataset,coco2014,object-detection,original",
         "script_variations": "validation,with-sample-ids,size.50",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -361,12 +401,12 @@
       }
     },
     {
-      "get,mlcommons,inference,src": {
+      "get,mlcommons,inference,src,_branch.dev": {
         "script_uid": "4b57186581024797",
         "script_alias": "get-mlperf-inference-src",
         "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
-        "script_variations": "",
-        "version": "master-git-c8c1e61f1a2164b5797504f1991d5c7aeea2364f",
+        "script_variations": "branch.dev",
+        "version": "custom-git-8d3b8ab09ac392b5a8656ad07d37fb8d7942595b",
         "parent": "generate-mlperf-inference-user-conf,3af4475745964b93"
       }
     },
@@ -397,7 +437,7 @@
         "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -407,57 +447,17 @@
         "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons",
         "script_variations": "",
         "version": "master",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
-      "get,mlcommons,inference,src": {
+      "get,mlcommons,inference,src,_branch.dev": {
         "script_uid": "4b57186581024797",
         "script_alias": "get-mlperf-inference-src",
         "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
-        "script_variations": "",
-        "version": "master-git-c8c1e61f1a2164b5797504f1991d5c7aeea2364f",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
-      }
-    },
-    {
-      "detect,os": {
-        "script_uid": "863735b7db8c44fc",
-        "script_alias": "detect-os",
-        "script_tags": "detect-os,detect,os,info",
-        "script_variations": "",
-        "version": "",
-        "parent": "get-mlperf-inference-src,4b57186581024797 ( branch.dev )"
-      }
-    },
-    {
-      "get,python3": {
-        "script_uid": "d0b5dd74373f4a62",
-        "script_alias": "get-python3",
-        "script_tags": "get,python,python3,get-python,get-python3",
-        "script_variations": "",
-        "version": "3.10.12",
-        "parent": "get-mlperf-inference-src,4b57186581024797 ( branch.dev )"
-      }
-    },
-    {
-      "detect,os": {
-        "script_uid": "863735b7db8c44fc",
-        "script_alias": "detect-os",
-        "script_tags": "detect-os,detect,os,info",
-        "script_variations": "",
-        "version": "",
-        "parent": "get-git-repo,ed603e7292974f10 ( branch.dev,_repo.https://github.com/mlcommons/inference )"
-      }
-    },
-    {
-      "get,git,repo,_branch.dev,_repo.https://github.com/mlcommons/inference": {
-        "script_uid": "ed603e7292974f10",
-        "script_alias": "get-git-repo",
-        "script_tags": "get,git,repo,repository,clone",
-        "script_variations": "branch.dev,repo.https://github.com/mlcommons/inference",
-        "version": "",
-        "parent": "get-mlperf-inference-src,4b57186581024797 ( branch.dev )"
+        "script_variations": "branch.dev",
+        "version": "custom-git-8d3b8ab09ac392b5a8656ad07d37fb8d7942595b",
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -466,8 +466,8 @@
         "script_alias": "get-mlperf-inference-src",
         "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
         "script_variations": "branch.dev",
-        "version": "custom",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "version": "custom-git-8d3b8ab09ac392b5a8656ad07d37fb8d7942595b",
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -477,7 +477,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.psutil",
         "version": "6.1.0",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -487,7 +487,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.diffusers",
         "version": "0.30.3",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -497,7 +497,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.transformers",
         "version": "4.46.2",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -507,7 +507,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.torchvision",
         "version": "0.20.1",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -517,7 +517,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.accelerate",
         "version": "1.1.0",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -527,7 +527,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.torchmetrics",
         "version": "1.5.1",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -537,7 +537,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.torch-fidelity",
         "version": "0.3.0",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -547,7 +547,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.open_clip_torch",
         "version": "2.29.0",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -557,7 +557,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.opencv-python",
         "version": "4.10.0.84",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -567,7 +567,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.scipy",
         "version": "1.10.1",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     },
     {
@@ -607,7 +607,7 @@
         "script_tags": "mlperf,benchmark-mlperf",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_pytorch,_cuda,_float16 )"
       }
     }
   ]

diff --git a/...r-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/cpu_info.json b/...r-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/cpu_info.json
@@ -23,5 +23,5 @@
   "CM_HOST_CPU_NUMA_NODES": "1",
   "CM_HOST_CPU_TOTAL_LOGICAL_CORES": "48",
   "CM_HOST_MEMORY_CAPACITY": "192G",
-  "CM_HOST_DISK_CAPACITY": "6.9T"
+  "CM_HOST_DISK_CAPACITY": "11T"
 }