Skip to content

Commit

Permalink
Results from self hosted Github actions - NVIDIARTX4090
Browse files Browse the repository at this point in the history
  • Loading branch information
@arjunsuresh
arjunsuresh committed Nov 15, 2024
1 parent 45e8e5d commit f7fb3ec
Show file tree
Hide file tree
Showing 11 changed files with 459 additions and 468 deletions.
2 changes: 1 addition & 1 deletion .../measurements/arjun_spr-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
|---------------------|------------|----------------------|--------------|-------------------|
| stable-diffusion-xl | offline | (16.3689, 237.82579) | 0.384 | - |
| stable-diffusion-xl | offline | (16.3689, 237.82579) | 0.173 | - |
13 changes: 2 additions & 11 deletions ...rence-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ pip install -U cmind

cm rm cache -f

cm pull repo gateoverflow@cm4mlops --checkout=555d6b8a42e85c354297330df5ff31bab3d3faa6
cm pull repo gateoverflow@cm4mlops --checkout=fed53ef8889585647e75df9d1f07fae97b2250c5

cm run script \
--tags=app,mlperf,inference,generic,_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline \
Expand Down Expand Up @@ -60,22 +60,13 @@ cm run script \
--env.CM_OUTPUT_FOLDER_NAME=test_results \
--env.CM_DOCKER_REUSE_EXISTING_CONTAINER=no \
--env.CM_DOCKER_DETACHED_MODE=yes \
--add_deps_recursive.mlperf-implementation.tags=_branch.dev \
--add_deps_recursive.get-mlperf-inference-results-dir.tags=_version.r4_1-dev \
--add_deps_recursive.get-mlperf-inference-submission-dir.tags=_version.r4_1-dev \
--add_deps_recursive.mlperf-inference-nvidia-scratch-space.tags=_version.r4_1-dev \
--add_deps_recursive.submission-checker.tags=_short-run \
--add_deps_recursive.coco2014-preprocessed.tags=_size.50,_with-sample-ids \
--add_deps_recursive.coco2014-dataset.tags=_size.50,_with-sample-ids \
--add_deps_recursive.nvidia-preprocess-data.extra_cache_tags=scc24-base \
--adr.mlperf-implementation.tags=_branch.dev \
--adr.get-mlperf-inference-results-dir.tags=_version.r4_1-dev \
--adr.get-mlperf-inference-submission-dir.tags=_version.r4_1-dev \
--adr.mlperf-inference-nvidia-scratch-space.tags=_version.r4_1-dev \
--adr.submission-checker.tags=_short-run \
--adr.coco2014-preprocessed.tags=_size.50,_with-sample-ids \
--adr.coco2014-dataset.tags=_size.50,_with-sample-ids \
--adr.nvidia-preprocess-data.extra_cache_tags=scc24-base \
--v=False \
--print_env=False \
--print_deps=False \
Expand Down Expand Up @@ -105,4 +96,4 @@ Model Precision: fp32
`FID_SCORE`: `237.82579`, Required accuracy for closed division `>= 23.01086` and `<= 23.95008`

### Performance Results
`Samples per second`: `0.383506`
`Samples per second`: `0.172851`
142 changes: 71 additions & 71 deletions ...ence-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/cm-version-info.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,53 @@
"parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline )"
}
},
{
"detect,os": {
"script_uid": "863735b7db8c44fc",
"script_alias": "detect-os",
"script_tags": "detect-os,detect,os,info",
"script_variations": "",
"version": "",
"parent": "get-mlperf-inference-src,4b57186581024797"
}
},
{
"get,python3": {
"script_uid": "d0b5dd74373f4a62",
"script_alias": "get-python3",
"script_tags": "get,python,python3,get-python,get-python3",
"script_variations": "",
"version": "3.10.12",
"parent": "get-mlperf-inference-src,4b57186581024797"
}
},
{
"detect,os": {
"script_uid": "863735b7db8c44fc",
"script_alias": "detect-os",
"script_tags": "detect-os,detect,os,info",
"script_variations": "",
"version": "",
"parent": "get-git-repo,ed603e7292974f10 ( branch.master,_repo.https://github.com/mlcommons/inference )"
}
},
{
"get,git,repo,_branch.master,_repo.https://github.com/mlcommons/inference": {
"script_uid": "ed603e7292974f10",
"script_alias": "get-git-repo",
"script_tags": "get,git,repo,repository,clone",
"script_variations": "branch.master,repo.https://github.com/mlcommons/inference",
"version": "",
"parent": "get-mlperf-inference-src,4b57186581024797"
}
},
{
"get,mlcommons,inference,src": {
"script_uid": "4b57186581024797",
"script_alias": "get-mlperf-inference-src",
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
"script_variations": "",
"version": "master-git-c8c1e61f1a2164b5797504f1991d5c7aeea2364f",
"version": "master-git-e398b90e85f9e64d34d6c67fe4b04d6cb7145c98",
"parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline )"
}
},
Expand All @@ -46,7 +86,7 @@
"script_alias": "get-mlperf-inference-src",
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
"script_variations": "",
"version": "master-git-c8c1e61f1a2164b5797504f1991d5c7aeea2364f",
"version": "master-git-e398b90e85f9e64d34d6c67fe4b04d6cb7145c98",
"parent": "get-mlperf-inference-utils,e341e5f86d8342e5"
}
},
Expand Down Expand Up @@ -227,7 +267,7 @@
"script_tags": "detect-os,detect,os,info",
"script_variations": "",
"version": "",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -247,7 +287,7 @@
"script_tags": "detect,cpu,detect-cpu,info",
"script_variations": "",
"version": "",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -257,7 +297,7 @@
"script_tags": "get,sys-utils-cm",
"script_variations": "",
"version": "",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -267,7 +307,7 @@
"script_tags": "get,python,python3,get-python,get-python3",
"script_variations": "",
"version": "3.10.12",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -277,7 +317,7 @@
"script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d",
"script_variations": "cudnn",
"version": "12.4",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -287,7 +327,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "torch_cuda",
"version": "2.5.1",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -297,7 +337,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "torchvision_cuda",
"version": "0.20.1",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -307,7 +347,7 @@
"script_tags": "get,raw,ml-model,stable-diffusion,sdxl,text-to-image",
"script_variations": "pytorch,fp16",
"version": "",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -317,7 +357,7 @@
"script_tags": "get,dataset,coco2014,object-detection,original",
"script_variations": "validation,with-sample-ids,size.50",
"version": "",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand Down Expand Up @@ -366,7 +406,7 @@
"script_alias": "get-mlperf-inference-src",
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
"script_variations": "",
"version": "master-git-c8c1e61f1a2164b5797504f1991d5c7aeea2364f",
"version": "master-git-e398b90e85f9e64d34d6c67fe4b04d6cb7145c98",
"parent": "generate-mlperf-inference-user-conf,3af4475745964b93"
}
},
Expand Down Expand Up @@ -397,7 +437,7 @@
"script_tags": "generate,mlperf,inference,user-conf,inference-user-conf",
"script_variations": "",
"version": "",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -407,7 +447,7 @@
"script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons",
"script_variations": "",
"version": "master",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -416,58 +456,18 @@
"script_alias": "get-mlperf-inference-src",
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
"script_variations": "",
"version": "master-git-c8c1e61f1a2164b5797504f1991d5c7aeea2364f",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
}
},
{
"detect,os": {
"script_uid": "863735b7db8c44fc",
"script_alias": "detect-os",
"script_tags": "detect-os,detect,os,info",
"script_variations": "",
"version": "",
"parent": "get-mlperf-inference-src,4b57186581024797 ( branch.dev )"
}
},
{
"get,python3": {
"script_uid": "d0b5dd74373f4a62",
"script_alias": "get-python3",
"script_tags": "get,python,python3,get-python,get-python3",
"script_variations": "",
"version": "3.10.12",
"parent": "get-mlperf-inference-src,4b57186581024797 ( branch.dev )"
"version": "master-git-e398b90e85f9e64d34d6c67fe4b04d6cb7145c98",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
"detect,os": {
"script_uid": "863735b7db8c44fc",
"script_alias": "detect-os",
"script_tags": "detect-os,detect,os,info",
"script_variations": "",
"version": "",
"parent": "get-git-repo,ed603e7292974f10 ( branch.dev,_repo.https://github.com/mlcommons/inference )"
}
},
{
"get,git,repo,_branch.dev,_repo.https://github.com/mlcommons/inference": {
"script_uid": "ed603e7292974f10",
"script_alias": "get-git-repo",
"script_tags": "get,git,repo,repository,clone",
"script_variations": "branch.dev,repo.https://github.com/mlcommons/inference",
"version": "",
"parent": "get-mlperf-inference-src,4b57186581024797 ( branch.dev )"
}
},
{
"get,mlcommons,inference,src,_branch.dev": {
"get,mlcommons,inference,src": {
"script_uid": "4b57186581024797",
"script_alias": "get-mlperf-inference-src",
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
"script_variations": "branch.dev",
"version": "custom",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"script_variations": "",
"version": "master-git-e398b90e85f9e64d34d6c67fe4b04d6cb7145c98",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -477,7 +477,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.psutil",
"version": "6.1.0",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -487,7 +487,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.diffusers",
"version": "0.30.3",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -497,7 +497,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.transformers",
"version": "4.46.2",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -507,7 +507,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.torchvision",
"version": "0.20.1",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -517,7 +517,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.accelerate",
"version": "1.1.0",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -527,7 +527,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.torchmetrics",
"version": "1.5.1",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -537,7 +537,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.torch-fidelity",
"version": "0.3.0",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -547,7 +547,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.open_clip_torch",
"version": "2.29.0",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -557,7 +557,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.opencv-python",
"version": "4.10.0.84",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand All @@ -567,7 +567,7 @@
"script_tags": "get,install,generic,generic-python-lib",
"script_variations": "package.scipy",
"version": "1.10.1",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
},
{
Expand Down Expand Up @@ -607,7 +607,7 @@
"script_tags": "mlperf,benchmark-mlperf",
"script_variations": "",
"version": "",
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cuda,_offline,_pytorch,_sdxl,_float16 )"
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_sdxl,_cuda,_float16 )"
}
}
]
Expand Down
2 changes: 1 addition & 1 deletion ...pr-reference-gpu-pytorch-v2.5.1-scc24-base_cu124/stable-diffusion-xl/offline/os_info.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,5 @@
],
"CM_HOST_PLATFORM_FLAVOR": "x86_64",
"CM_HOST_PYTHON_BITS": "64",
"CM_HOST_SYSTEM_NAME": "1dc3ae3f2533"
"CM_HOST_SYSTEM_NAME": "f47e80bfc592"
}
Loading

0 comments on commit f7fb3ec

Please sign in to comment.