Added support for automotive pointpainting benchmark (#172)

* initial commit for pointpillar * rename benchmark name + add deps * [Automated Commit] Format Codebase * added scipy dependency * Revert "added scipy dependency" This reverts commit 3b3a0e8. * add scipy dependency * version max as string * changes for registering to docker cache * added opencv dependency * add more dependencies * support custom query count for pointpainting * fix version check ffmpeg * version dlted * remove scipy dependencu * fixed scipy version * version change - test commit * integrated accuracy script * add extra loadgen options * fix for accuracy * added starting weights filename * commit for valid perf run * submission checker version defaulted to 5.0 * fix version bug * commit for compliance * add path to new env * add test 4 for pointpainting * commit to test numpy fix * add dep name for scipy * fix name keys
mlcommons · Jan 31, 2025 · 7ee9cf4 · 7ee9cf4
1 parent 1dd7264
commit 7ee9cf4
Show file tree

Hide file tree

Showing 12 changed files with 211 additions and 3 deletions.
diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py
@@ -68,7 +68,7 @@ def preprocess(i):
             str(env['MLC_MLPERF_LOADGEN_BATCH_SIZE'])
 
     if env.get('MLC_MLPERF_LOADGEN_QUERY_COUNT', '') != '' and not env.get('MLC_TMP_IGNORE_MLPERF_QUERY_COUNT', False) and (
-            env['MLC_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['MLC_MODEL'] or 'llama2' in env['MLC_MODEL'] or 'mixtral' in env['MLC_MODEL'] or 'llama3' in env['MLC_MODEL']) and env.get('MLC_MLPERF_RUN_STYLE', '') != "valid":
+            env['MLC_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['MLC_MODEL'] or 'llama2' in env['MLC_MODEL'] or 'mixtral' in env['MLC_MODEL'] or 'llama3' in env['MLC_MODEL'] or 'pointpainting' in env['MLC_MODEL']) and (env.get('MLC_MLPERF_RUN_STYLE', '') != "valid" or 'pointpainting' in env['MLC_MODEL']):
         env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --count " + \
             env['MLC_MLPERF_LOADGEN_QUERY_COUNT']
 
@@ -524,6 +524,28 @@ def get_run_cmd_reference(
         cmd = cmd.replace("--count", "--total-sample-count")
         cmd = cmd.replace("--max-batchsize", "--batch-size")
 
+    elif "pointpainting" in env['MLC_MODEL']:
+        env['RUN_DIR'] = os.path.join(
+            env['MLC_MLPERF_INFERENCE_SOURCE'],
+            "automotive",
+            "3d-object-detection")
+
+        cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " \
+            " --dataset waymo" + \
+            " --dataset-path " + env['MLC_DATASET_WAYMO_PATH'] + \
+            " --lidar-path " + env['MLC_ML_MODEL_POINT_PILLARS_PATH'] + \
+            " --segmentor-path " + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] + \
+            " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \
+            " --output " + env['MLC_MLPERF_OUTPUT_DIR'] + \
+            " --dtype " + env['MLC_MLPERF_MODEL_PRECISION'].replace("float", "fp") + \
+            scenario_extra_options + \
+            env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + mode_extra_options
+
+        if env.get('MLC_MLPERF_POINTPAINTING_TIME', '') != '':
+            cmd += f" --time {env['MLC_MLPERF_POINTPAINTING_TIME']}"
+
+        print(cmd)
+
     if env.get('MLC_NETWORK_LOADGEN', '') in ["lon", "sut"]:
         cmd = cmd + " " + "--network " + env['MLC_NETWORK_LOADGEN']
         if env.get('MLC_NETWORK_LOADGEN_SUT_SERVERS', []):

diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml
@@ -66,6 +66,9 @@ input_mapping:
   multistream_target_latency: MLC_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY
   network: MLC_NETWORK_LOADGEN
   sut_servers: MLC_NETWORK_LOADGEN_SUT_SERVERS
+  pointpillars_checkpoint_path: MLC_ML_MODEL_POINT_PILLARS_PATH
+  deeplab_resnet50_path: MLC_ML_MODEL_DPLAB_RESNET50_PATH
+  waymo_path: MLC_DATASET_WAYMO_PATH
 
 # Duplicate CM environment variables to the ones used in native apps
 env_key_mappings:
@@ -491,7 +494,6 @@ deps:
   - tags: get,ml-model,llama3
     names:
       - llama3-405b-model
-      - llama3-402b-model
     enable_if_env:
       MLC_MODEL:
         - llama3_1-405b
@@ -502,6 +504,26 @@ deps:
       MLC_RUN_STATE_DOCKER:
         - "yes"
 
+  ## pointpainting
+  - tags: get,ml-model,pointpillars
+    names:
+      - pointpillars-model
+    enable_if_env:
+      MLC_MODEL:
+        - pointpainting
+    skip_if_env:
+      MLC_RUN_STATE_DOCKER:
+        - "yes"
+  - tags: get,ml-model,resnet50-deeplab
+    enable_if_env:
+      MLC_MODEL:
+        - pointpainting
+    skip_if_env:
+      MLC_RUN_STATE_DOCKER:
+        - "yes"
+    names:
+      - resnet50-deeplab-model
+
   ########################################################################
   # Install datasets
 
@@ -641,6 +663,17 @@ deps:
       MLC_USE_DATASET_FROM_HOST:
         - "yes"
 
+  ## waymo for pointpillats
+  - tags: get,dataset,waymo
+    enable_if_env:
+      MLC_MODEL:
+        - pointpainting
+    skip_if_env:
+      MLC_RUN_STATE_DOCKER:
+        - "yes"
+    names:
+      - waymo-dataset
+
   ## llama3_1 dataset
   - tags: get,dataset,mlperf,inference,llama3,_validation
     names:
@@ -1299,6 +1332,45 @@ variations:
       MLC_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<<MLC_TORCH_VERSION>>>+cpu.html"
       MLC_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL_DGL: "https://data.dgl.ai/wheels/torch-<<<MLC_TORCH_VERSION_MAJOR_MINOR>>>/repo.html"
 
+  pointpainting:
+    group: models
+    env:
+      MLC_MODEL: pointpainting
+      MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://github.com/mlcommons/mlperf-automations/tree/dev/script/get-ml-model-pointpainting,https://github.com/mlcommons/mlperf-automations/tree/dev/script/get-ml-model-resnet50-deeplab"
+    adr:
+      pytorch:
+        version_max: "2.2.2"
+      torchvision:
+        version_max: "0.17.2"
+    deps:
+      - tags: get,generic-python-lib,_package.shapely
+      - tags: get,generic-python-lib,_package.numba
+      - tags: get,generic-python-lib,_package.open3d
+      - tags: get,generic-python-lib,_package.numpy
+        version_max: "1.26.4"
+        names: 
+          - numpy
+      - tags: get,generic-python-lib,_package.numpy
+        version_max: "2.0.2"
+        names: 
+          - numpy-upgrade
+      - tags: get,generic-python-lib,_package.numpy
+        version_max: "1.26.4"
+        names: 
+          - numpy-downgrade
+      - tags: get,generic-python-lib,_package.tensorboard
+      - tags: get,generic-python-lib,_package.onnxruntime
+      - tags: get,generic-python-lib,_package.opencv-python
+      - tags: get,generic-python-lib,_package.scikit-image
+      - tags: get,generic-python-lib,_package.scipy
+        version_max: "1.11.2"
+        names: 
+          - scipy
+      - tags: get,generic-python-lib,_package.ninja
+      - tags: get,generic-sys-util,_ffmpeg
+      - tags: get,generic-sys-util,_libsm6
+      - tags: get,generic-sys-util,_libxext6
+
   llama3_1-405b:
     group: models
     env:

diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py
@@ -129,6 +129,13 @@ def postprocess(i):
             accuracy_log_file_option_name = " --mlperf-accuracy-file "
             datatype_option = " --dtype " + env['MLC_IMAGENET_ACCURACY_DTYPE']
 
+        elif model == "pointpainting":
+            accuracy_filename = "accuracy-waymo.py"
+            accuracy_filepath = os.path.join(
+                env['MLC_MLPERF_INFERENCE_POINTPAINTING_PATH'], accuracy_filename)
+            accuracy_log_file_option_name = " --mlperf-accuracy-file "
+            datatype_option = ""
+
         elif model == "retinanet":
             accuracy_filename = "accuracy-openimages.py"
             accuracy_filepath = os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools",

diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml
@@ -261,6 +261,10 @@ variations:
   reference,rgat:
     default_variations:
       backend: pytorch
+
+  reference,pointpainting:
+    default_variations:
+      backend: pytorch
 
   reference,sdxl_:
     default_variations:
@@ -823,6 +827,51 @@ variations:
           - igbh-original
           - igbh-dataset
 
+  pointpainting:
+    group:
+      model
+    add_deps_recursive:
+      mlperf-inference-implementation:
+        tags: _pointpainting
+    env:
+      MLC_MODEL:
+        pointpainting
+    docker:
+      deps:
+      - tags: get,dataset,waymo
+        enable_if_env:
+          MLC_USE_DATASET_FROM_HOST:
+          - 'yes'
+        names:
+          - waymo-dataset
+      - tags: get,ml-model,pointpillars
+        enable_if_env:
+          MLC_USE_DATASET_FROM_HOST:
+          - 'yes'
+        names:
+          - pointpillars-model
+      - tags: get,ml-model,resnet50-deeplab
+        enable_if_env:
+          MLC_USE_DATASET_FROM_HOST:
+          - 'yes'
+        names:
+          - resnet50-deeplab-model
+    posthook_deps:
+    - enable_if_env:
+        MLC_MLPERF_LOADGEN_MODE:
+        - accuracy
+        - all
+        MLC_MLPERF_ACCURACY_RESULTS_DIR:
+        - 'on'
+      skip_if_env:
+        MLC_MLPERF_IMPLEMENTATION:
+        - nvidia
+      names:
+      - mlperf-accuracy-script
+      - waymo-accuracy-script
+      tags: run,accuracy,mlperf,_waymo
+
+
   llama3_1-405b:
     group:
       model
@@ -1904,6 +1953,9 @@ docker:
    - "${{ MLC_DATASET_KITS19_PREPROCESSED_PATH }}:${{ MLC_DATASET_KITS19_PREPROCESSED_PATH }}"
    - "${{ MLC_DATASET_IGBH_PATH }}:${{ MLC_DATASET_IGBH_PATH }}"
    - "${{ MLC_ML_MODEL_RGAT_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_RGAT_CHECKPOINT_PATH }}"
+   - "${{ MLC_DATASET_WAYMO_PATH }}:${{ MLC_DATASET_WAYMO_PATH }}"
+   - "${{ MLC_ML_MODEL_POINT_PILLARS_PATH }}:${{ MLC_ML_MODEL_POINT_PILLARS_PATH }}"
+   - "${{ MLC_ML_MODEL_DPLAB_RESNET50_PATH }}:${{ MLC_ML_MODEL_DPLAB_RESNET50_PATH }}" 
   skip_run_cmd: 'no'
   shm_size: '32gb'
   interactive: True

diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml
@@ -95,6 +95,43 @@ variations:
         brew: ''
         dnf: dmidecode
         yum: dmidecode
+  ffmpeg:
+    env:
+      MLC_SYS_UTIL_NAME: ffmpeg
+      MLC_SYS_UTIL_VERSION_CMD: ffmpeg -version # tbd: regular expression for version
+      MLC_TMP_VERSION_DETECT_GROUP_NUMBER: 0
+    new_env_keys:
+    - MLC_FFMPEG_VERSION
+    state:
+      ffmpeg: # tbd: complete for other flavours of linux
+        apt: ffmpeg
+        brew: ''
+        dnf: ''   
+        yum: ''
+  libsm6:
+    env:
+      MLC_SYS_UTIL_NAME: libsm6 # tbd: regular expression for version as well as whether its installed?
+      MLC_TMP_VERSION_DETECT_GROUP_NUMBER: 0
+    new_env_keys:
+    - MLC_LIBSM6_VERSION
+    state:
+      libsm6: # tbd: complete for other flavours of linux
+        apt: libsm6
+        brew: ''
+        dnf: ''   
+        yum: ''
+  libxext6:
+    env:
+      MLC_SYS_UTIL_NAME: libxext6 # tbd: regular expression for version as well as whether its installed?
+      MLC_TMP_VERSION_DETECT_GROUP_NUMBER: 0
+    new_env_keys:
+    - MLC_LIBEXT6_VERSION
+    state:
+      libxext6: # tbd: complete for other flavours of linux
+        apt: libxext6
+        brew: ''
+        dnf: ''   
+        yum: ''
   g++-11:
     env:
       MLC_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE: 'yes'

diff --git a/script/get-mlperf-inference-src/customize.py b/script/get-mlperf-inference-src/customize.py
@@ -114,6 +114,8 @@ def postprocess(i):
         inference_root, 'graph', 'R-GAT')
     env['MLC_MLPERF_INFERENCE_3DUNET_PATH'] = os.path.join(
         inference_root, 'vision', 'medical_imaging', '3d-unet-kits19')
+    env['MLC_MLPERF_INFERENCE_POINTPAINTING_PATH'] = os.path.join(
+        inference_root, 'automotive', '3d-object-detection')
 
     env['MLC_GET_DEPENDENT_CACHED_PATH'] = inference_root
 

diff --git a/script/get-mlperf-inference-src/meta.yaml b/script/get-mlperf-inference-src/meta.yaml
@@ -30,6 +30,7 @@ new_env_keys:
 - MLC_MLPERF_INFERENCE_VERSION
 - MLC_MLPERF_INFERENCE_VISION_PATH
 - MLC_MLPERF_LAST_RELEASE
+- MLC_MLPERF_INFERENCE_POINTPAINTING_PATH
 - +PYTHONPATH
 prehook_deps:
 - env:

diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py
@@ -202,6 +202,10 @@ def preprocess(i):
             CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "llama3.1-405b", "evaluate-accuracy.py") + "' --checkpoint-path '" + env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(
                 result_dir, "mlperf_log_accuracy.json") + "' --dtype '" + env['MLC_ACCURACY_DTYPE'] + "' --dataset-file '" + env['MLC_DATASET_LLAMA3_PATH'] + "' > '" + out_file + "'"
 
+        elif dataset == "waymo":
+            CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "automotive", "3d-object-detection", "accuracy_waymo.py") + "' --mlperf-accuracy-file '" + os.path.join(
+                result_dir, "mlperf_log_accuracy.json") + "' --waymo-dir '" + env['MLC_DATASET_WAYMO_PATH'] + "' > '" + out_file + "'"
+
         else:
             return {'return': 1, 'error': 'Unsupported dataset'}
 

diff --git a/script/process-mlperf-accuracy/meta.yaml b/script/process-mlperf-accuracy/meta.yaml
@@ -269,3 +269,7 @@ variations:
     env:
       MLC_DATASET: dataset_llama3
     group: dataset
+  waymo:
+    env:
+      MLC_DATASET: waymo
+    group: dataset
diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py
@@ -128,6 +128,9 @@ def preprocess(i):
         test_list.remove("TEST01")
         # test_list.remove("TEST05")
 
+    if "pointpainting" in env["MLC_MODEL"].lower():
+        test_list.append("TEST04")
+
     if "llama2" in env['MLC_MODEL'].lower(
     ) or "mixtral-8x7b" in env['MLC_MODEL']:
         test_list.append("TEST06")

diff --git a/script/run-mlperf-inference-app/meta.yaml b/script/run-mlperf-inference-app/meta.yaml
@@ -119,6 +119,9 @@ input_mapping:
   use_dataset_from_host: MLC_USE_DATASET_FROM_HOST
   use_model_from_host: MLC_USE_MODEL_FROM_HOST
   rgat_checkpoint_path: RGAT_CHECKPOINT_PATH
+  pointpillars_checkpoint_path: MLC_ML_MODEL_POINT_PILLARS_PATH
+  deeplab_resnet50_path: MLC_ML_MODEL_DPLAB_RESNET50_PATH
+  waymo_path: MLC_DATASET_WAYMO_PATH
 
 new_state_keys:
 - app_mlperf_inference_*
@@ -466,6 +469,7 @@ input_description:
     - efficientnet
     - rgat
     - llama3_1-405b
+    - pointpainting
     default: resnet50
     desc: MLPerf model
     sort: 200

diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py
@@ -11,7 +11,7 @@ def preprocess(i):
 
     submission_dir = env.get("MLC_MLPERF_INFERENCE_SUBMISSION_DIR", "")
 
-    version = env.get('MLC_MLPERF_SUBMISSION_CHECKER_VERSION', '')
+    version = env.get('MLC_MLPERF_SUBMISSION_CHECKER_VERSION', 'v5.0')
 
     if submission_dir == "":
         return {'return': 1,