Results on mlc-server added by MLCommons

mlcommons · Jan 30, 2025 · a3f9cd9 · a3f9cd9
1 parent 68aea56
commit a3f9cd9
Show file tree

Hide file tree

Showing 25 changed files with 16,307 additions and 0 deletions.
diff --git a/closed/MLCommons/code/pointpainting/README.md b/closed/MLCommons/code/pointpainting/README.md
@@ -0,0 +1 @@
+TBD
diff --git a/closed/MLCommons/systems/mlc-server-reference-gpu-pytorch_v2.2.2-cu124.json b/closed/MLCommons/systems/mlc-server-reference-gpu-pytorch_v2.2.2-cu124.json
@@ -0,0 +1,37 @@
+{
+  "accelerator_frequency": "1980000 MHz",
+  "accelerator_host_interconnect": "N/A",
+  "accelerator_interconnect": "N/A",
+  "accelerator_interconnect_topology": "",
+  "accelerator_memory_capacity": "79.09661865234375 GB",
+  "accelerator_memory_configuration": "N/A",
+  "accelerator_model_name": "NVIDIA H100 80GB HBM3",
+  "accelerator_on-chip_memories": "",
+  "accelerators_per_node": 8,
+  "cooling": "air",
+  "division": "closed",
+  "framework": "pytorch v2.2.2",
+  "host_memory_capacity": "2.1T",
+  "host_memory_configuration": "undefined",
+  "host_network_card_count": "1",
+  "host_networking": "Gig Ethernet",
+  "host_networking_topology": "N/A",
+  "host_processor_caches": "L1d cache: 5.3 MiB (112 instances), L1i cache: 3.5 MiB (112 instances), L2 cache: 224 MiB (112 instances), L3 cache: 210 MiB (2 instances)",
+  "host_processor_core_count": "56",
+  "host_processor_frequency": "2001.0000",
+  "host_processor_interconnect": "",
+  "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8480+",
+  "host_processors_per_node": "2",
+  "host_storage_capacity": "19T",
+  "host_storage_type": "SSD",
+  "hw_notes": "",
+  "number_of_nodes": "1",
+  "operating_system": "Ubuntu 22.04 (linux-5.15.0-113-generic-glibc2.35)",
+  "other_software_stack": "Python: 3.10.12, LLVM-15.0.6 , CUDA 12.4",
+  "status": "available",
+  "submitter": "MLCommons",
+  "sw_notes": "",
+  "system_name": "34127abc9663",
+  "system_type": "edge",
+  "system_type_detail": "edge server"
+}
diff --git a/open/MLCommons/code/pointpainting/README.md b/open/MLCommons/code/pointpainting/README.md
@@ -0,0 +1 @@
+TBD
diff --git a/...-server-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/README.md b/...-server-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/README.md
@@ -0,0 +1,43 @@
+*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.*
+
+## Host platform
+
+* OS version: Linux-5.15.0-113-generic-x86_64-with-glibc2.35
+* CPU version: x86_64
+* Python version: 3.10.12 (main, Jan 17 2025, 14:35:34) [GCC 11.4.0]
+* MLC version: 0.1.0
+
+## CM Run Command
+
+See [CM installation guide](https://docs.mlcommons.org/inference/install/).
+
+```bash
+pip install -U mlcflow
+
+mlc rm cache -f
+
+mlc pull repo anandhu-eng@mlperf-automations --checkout=4192cdb677ef913c863abd7734fe821ec6c46793
+
+
+```
+*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf,
+ you should simply reload anandhu-eng@mlperf-automations without checkout and clean MLC cache as follows:*
+
+```bash
+mlc rm repo anandhu-eng@mlperf-automations
+mlc pull repo anandhu-eng@mlperf-automations
+mlc rm cache -f
+
+```
+
+## Results
+
+Platform: mlc-server-reference-gpu-pytorch_v2.2.2-cu124
+
+Model Precision: fp32
+
+### Accuracy Results 
+`mAP`: `54.24745`, Required accuracy for closed division `>= 0.54196`
+
+### Performance Results 
+`90th percentile latency (ns)`: `1568895029.0`
diff --git a/...server-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/accuracy_console.out b/...server-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/accuracy_console.out
@@ -0,0 +1,5 @@
+INFO:main:Namespace(dataset='waymo', dataset_path='/mlc-mount/data/common/rod/waymo/kitti_format', profile=None, scenario='SingleStream', max_batchsize=1, threads=1, accuracy=True, find_peak_performance=False, backend='pytorch', model_name='pointpainting', output='/root/MLC/repos/local/cache/get-mlperf-inference-results-dir_fc658933/valid_results/34127abc9663-reference-gpu-pytorch-v2.2.2-cu124/pointpainting/singlestream/accuracy', qps=None, lidar_path='/mlc-mount/data/common/rod/waymo/pp_ep36.pth', segmentor_path='/mlc-mount/data/common/rod/waymo/best_deeplabv3plus_resnet50_waymo_os16.pth', dtype='fp32', device='cuda', user_conf='/root/MLC/repos/anandhu-eng@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/9b0967720c61400fa3f06e4f468e7dad.conf', audit_conf='audit.config', time=None, count=None, debug=False, performance_sample_count=5000, max_latency=None, samples_per_query=8)
+/root/venv/mlc/lib/python3.10/site-packages/torch/functional.py:507: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3549.)
+  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
+INFO:main:starting TestScenario.SingleStream
+Using Segmentation Network -- deeplabv3plus
diff --git a/...ts/mlc-server-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/cpu_info.json b/...ts/mlc-server-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/cpu_info.json
@@ -0,0 +1,27 @@
+{
+  "MLC_HOST_CPU_WRITE_PROTECT_SUPPORT": "yes",
+  "MLC_HOST_CPU_MICROCODE": "0x2b0005c0",
+  "MLC_HOST_CPU_FPU_SUPPORT": "yes",
+  "MLC_HOST_CPU_FPU_EXCEPTION_SUPPORT": "yes",
+  "MLC_HOST_CPU_BUGS": "coma tlb_mmatch spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi",
+  "MLC_HOST_CPU_TLB_SIZE": "Not Found",
+  "MLC_HOST_CPU_CFLUSH_SIZE": "64",
+  "MLC_HOST_CPU_ARCHITECTURE": "x86_64",
+  "MLC_HOST_CPU_TOTAL_CORES": "224",
+  "MLC_HOST_CPU_ON_LINE_CPUS_LIST": "0-223",
+  "MLC_HOST_CPU_VENDOR_ID": "GenuineIntel",
+  "MLC_HOST_CPU_MODEL_NAME": "Intel(R) Xeon(R) Platinum 8480+",
+  "MLC_HOST_CPU_FAMILY": "6",
+  "MLC_HOST_CPU_THREADS_PER_CORE": "2",
+  "MLC_HOST_CPU_PHYSICAL_CORES_PER_SOCKET": "56",
+  "MLC_HOST_CPU_SOCKETS": "2",
+  "MLC_HOST_CPU_MAX_MHZ": "2001.0000",
+  "MLC_HOST_CPU_L1D_CACHE_SIZE": "5.3 MiB (112 instances)",
+  "MLC_HOST_CPU_L1I_CACHE_SIZE": "3.5 MiB (112 instances)",
+  "MLC_HOST_CPU_L2_CACHE_SIZE": "224 MiB (112 instances)",
+  "MLC_HOST_CPU_L3_CACHE_SIZE": "210 MiB (2 instances)",
+  "MLC_HOST_CPU_NUMA_NODES": "2",
+  "MLC_HOST_CPU_TOTAL_LOGICAL_CORES": "224",
+  "MLC_HOST_MEMORY_CAPACITY": "2.1T",
+  "MLC_HOST_DISK_CAPACITY": "19T"
+}
diff --git a/...nts/mlc-server-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/mlc-deps.mmd b/...nts/mlc-server-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/mlc-deps.mmd
@@ -0,0 +1,79 @@
+graph TD
+    app-mlperf-inference,d775cac873ee4231_(_reference,_pointpainting,_pytorch,_cuda,_valid,_r5.0-dev_default,_singlestream_) --> detect,os
+    app-mlperf-inference,d775cac873ee4231_(_reference,_pointpainting,_pytorch,_cuda,_valid,_r5.0-dev_default,_singlestream_) --> get,sys-utils-cm
+    app-mlperf-inference,d775cac873ee4231_(_reference,_pointpainting,_pytorch,_cuda,_valid,_r5.0-dev_default,_singlestream_) --> get,python
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> detect,os
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> get,python3
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> get,git,repo,_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git
+    app-mlperf-inference,d775cac873ee4231_(_reference,_pointpainting,_pytorch,_cuda,_valid,_r5.0-dev_default,_singlestream_) --> get,mlcommons,inference,src,_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> detect,os
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> get,python3
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> get,git,repo,_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git
+    get-mlperf-inference-utils,e341e5f86d8342e5 --> get,mlperf,inference,src,_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git
+    app-mlperf-inference,d775cac873ee4231_(_reference,_pointpainting,_pytorch,_cuda,_valid,_r5.0-dev_default,_singlestream_) --> get,mlperf,inference,utils
+    get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,cuda,_toolkit
+    get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,python3
+    get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3
+    get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,generic-python-lib,_package.pycuda
+    get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> get,python3
+    get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,generic-python-lib,_package.numpy
+    app-mlperf-inference,d775cac873ee4231_(_reference,_pointpainting,_pytorch,_cuda,_valid,_r5.0-dev_default,_singlestream_) --> get,cuda-devices,_with-pycuda
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> detect,cpu
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,sys-utils-cm
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,python
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,cuda,_cudnn
+    get-generic-python-lib,94b62a682bc44791_(_torch_cuda_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_torch_cuda
+    get-generic-python-lib,94b62a682bc44791_(_torchvision_cuda_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_torchvision_cuda
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,ml-model,pointpillars
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,ml-model,resnet50-deeplab
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,dataset,waymo
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,cpu
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> get,python
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> detect,os
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> get,python3
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> get,git,repo,_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> get,mlcommons,inference,src,_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git
+    get-mlperf-inference-sut-configs,c2fbf72009e2445b --> get,cache,dir,_name.mlperf-inference-sut-configs
+    generate-mlperf-inference-user-conf,3af4475745964b93 --> get,sut,configs
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> generate,user-conf,mlperf,inference
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,loadgen
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> detect,os
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> get,python3
+    get-mlperf-inference-src,4b57186581024797_(_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git_) --> get,git,repo,_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,mlcommons,inference,src,_branch.patch-16,_repo.https://github.com/anandhu-eng/inference.git
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,mlcommons,inference,src
+    get-generic-python-lib,94b62a682bc44791_(_package.psutil_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.psutil
+    get-generic-python-lib,94b62a682bc44791_(_package.shapely_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.shapely
+    get-generic-python-lib,94b62a682bc44791_(_package.numba_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.numba
+    get-generic-python-lib,94b62a682bc44791_(_package.open3d_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.open3d
+    get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.numpy
+    get-generic-python-lib,94b62a682bc44791_(_package.tensorboard_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.tensorboard
+    get-generic-python-lib,94b62a682bc44791_(_package.onnxruntime_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.onnxruntime
+    get-generic-python-lib,94b62a682bc44791_(_package.opencv-python_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.opencv-python
+    get-generic-python-lib,94b62a682bc44791_(_package.scikit-image_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.scikit-image
+    get-generic-python-lib,94b62a682bc44791_(_package.scipy_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.scipy
+    get-generic-python-lib,94b62a682bc44791_(_package.ninja_) --> get,python3
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-python-lib,_package.ninja
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-sys-util,_ffmpeg
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-sys-util,_libsm6
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> get,generic-sys-util,_libxext6
+    detect-cpu,586c8a43320142f7 --> detect,os
+    benchmark-program,19f369ef47084895 --> detect,cpu
+    benchmark-program-mlperf,cfff0132a8aa4018 --> benchmark-program,program
+    app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_cuda,_pytorch,_pointpainting,_singlestream,_fp32_) --> benchmark-mlperf
diff --git a/...rver-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/mlc-deps.png b/...rver-reference-gpu-pytorch_v2.2.2-cu124/pointpainting/singlestream/mlc-deps.png
diff --git a/...2.2.2-cu124/pointpainting/singlestream/mlc-server-reference-gpu-pytorch_v2.2.2-cu124.json b/...2.2.2-cu124/pointpainting/singlestream/mlc-server-reference-gpu-pytorch_v2.2.2-cu124.json
@@ -0,0 +1,7 @@
+{
+  "starting_weights_filename": "https://github.com/mlcommons/mlperf-automations/tree/dev/script/get-ml-model-pointpainting,https://github.com/mlcommons/mlperf-automations/tree/dev/script/get-ml-model-resnet50-deeplab",
+  "retraining": "no",
+  "input_data_types": "fp32",
+  "weight_data_types": "fp32",
+  "weight_transformations": "none"
+}