Update github tests to use mlc repo, mlc-scripts package name added

* Update test-mlperf-inference-resnet50.yml * Support nvidia implementation for v5.0 * Update and rename test-mlc-tutorial-tvm.yml to test-mlperf-inference-tvm-resnet50.yml * Update test-mlc-script-features.yml | Added retinanet test * Delete .github/workflows/test-mlc-tutorial-retinanet.yml * Use mlc-scripts name
mlcommons · Jan 25, 2025 · 7b0d537 · 7b0d537
1 parent e91a7b2
commit 7b0d537
Show file tree

Hide file tree

Showing 11 changed files with 73 additions and 122 deletions.
diff --git a/.github/workflows/test-mlc-script-features.yml b/.github/workflows/test-mlc-script-features.yml
@@ -70,3 +70,12 @@ jobs:
       run: |
         mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --image_name=mlc-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=local --quiet
 
+    - name: Run MLPerf Inference Retinanet with native and virtual Python
+      run: |
+        mlcr --tags=app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu  --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --scenario=Offline --mode=accuracy --test_query_count=10 --rerun --quiet
+        
+        mlcr --tags=app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu  --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --scenario=Offline --mode=performance --test_query_count=10 --rerun --quiet
+
+        mlcr --tags=install,python-venv --version=3.10.8 --name=mlperf --quiet
+
+        mlcr --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --adr.python.name=mlperf --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --submitter=Community --implementation=cpp --hw_name=default --model=retinanet --backend=onnxruntime --device=cpu --scenario=Offline --quiet
diff --git a/.github/workflows/test-mlc-tutorial-retinanet.yml b/.github/workflows/test-mlc-tutorial-retinanet.yml
diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml
@@ -27,6 +27,7 @@ jobs:
             backend: tf
           - os: windows-latest
             implementation: cpp
+
     steps:
     - uses: actions/checkout@v4
       with:
@@ -56,8 +57,9 @@ jobs:
     - name: Test MLPerf Inference ResNet50 (Linux/macOS)
       if: matrix.os != 'windows-latest'
       run: |
-        mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet 
+        mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet     
     - name: Retrieve secrets from Keeper
+      if: github.repository_owner == 'mlcommons'
       id: ksecrets
       uses: Keeper-Security/ksm-action@master
       with:
@@ -67,6 +69,7 @@ jobs:
     - name: Push Results
       env:
         GITHUB_TOKEN: ${{ env.PAT }}
+      if: github.repository_owner == 'mlcommons'
       run: |
         git config --global user.name "mlcommons-bot"
         git config --global user.email "[email protected]"
@@ -75,3 +78,15 @@ jobs:
         git config --global credential.https://gist.github.com.helper ""
         git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
         mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
+    - name: Push Results
+      env:
+        GITHUB_TOKEN: ${{ secrets.PAT1 }}
+      if: github.repository_owner == 'gateoverflow'
+      run: |
+        git config --global user.name "mlcommons-bot"
+        git config --global user.email "[email protected]"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
diff --git a/.github/workflows/test-mlc-tutorial-tvm.yml → ...ws/test-mlperf-inference-tvm-resnet50.yml b/.github/workflows/test-mlc-tutorial-tvm.yml → ...ws/test-mlperf-inference-tvm-resnet50.yml
@@ -1,16 +1,15 @@
-name: MLC tutorial tvm
+name: MLPerf Inference TVM ResNet50
 
 on:
   pull_request_target:
     branches: [ "main", "dev" ]
     paths:
-      - '.github/workflows/test-mlc-tutorial-tvm.yml'
+      - '.github/workflows/test-mlperf-inference-tvm-resnet50.yml'
       - '**'
       - '!**.md'
 
 jobs:
-  build:
-
+  tvm-run:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

diff --git a/.github/workflows/test-mlperf-inference-tvm.yml b/.github/workflows/test-mlperf-inference-tvm.yml
diff --git a/pyproject.toml b/pyproject.toml
@@ -3,7 +3,7 @@ requires = ["setuptools>=61.0", "wheel", "toml", "mlcflow"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "mlperf"
+name = "mlc-scripts"
 version = "0.0.1"
 description = "Automation scripts for running ML applications using MLC interface"
 authors = [
@@ -12,7 +12,7 @@ authors = [
 license = { file = "LICENSE.md" }
 readme = "README.md"
 requires-python = ">=3.7"
-keywords = ["mlc", "mlcflow", "pypi", "package", "automation", "mlperf", "mlcr"]
+keywords = ["mlc", "mlcscripts", "mlcflow", "pypi", "package", "automation", "mlperf", "mlcr"]
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: Apache Software License",

diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml
@@ -356,6 +356,27 @@ variations:
         env:
             MLC_ENV_NVMITTEN_DOCKER_WHEEL_PATH: '/opt/nvmitten-0.1.3b0-cp310-cp310-linux_aarch64.whl'
 
+  nvidia-original,r5.0-dev_default:
+    docker:
+      build_deps:
+        - tags: detect,os
+      image_name: mlperf-inference-nvidia-v5.0-dev-common
+    update_meta_if_env:
+      - enable_if_env:
+          MLC_HOST_PLATFORM_FLAVOR:
+            - x86_64
+        docker:
+          base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v4.0-cuda12.2-cudnn8.9-x86_64-ubuntu20.04-public
+        env:
+            MLC_ENV_NVMITTEN_DOCKER_WHEEL_PATH: '/opt/nvmitten-0.1.3b0-cp38-cp38-linux_x86_64.whl'
+
+      - skip_if_env:
+          MLC_HOST_PLATFORM_FLAVOR:
+            - x86_64
+        docker:
+          base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v4.0-cuda12.2-cudnn8.9-aarch64-ubuntu22.04-public
+        env:
+            MLC_ENV_NVMITTEN_DOCKER_WHEEL_PATH: '/opt/nvmitten-0.1.3b0-cp310-cp310-linux_aarch64.whl'
 
 
   nvidia-original,gptj_:
@@ -1716,10 +1737,10 @@ variations:
       reproducibility
     add_deps_recursive:
       nvidia-inference-common-code:
-        version: r4.1
+        version: r4.1-dev
         tags: _mlcommons
       nvidia-inference-server:
-        version: r4.1
+        version: r4.0
         tags: _mlcommons
       intel-harness:
         tags: _v4.1

diff --git a/script/build-mlperf-inference-server-nvidia/meta.yaml b/script/build-mlperf-inference-server-nvidia/meta.yaml
@@ -228,25 +228,9 @@ variations:
             - 8
 
 versions:
-  r2.1:
-    add_deps_recursive:
-      nvidia-inference-common-code:
-        version: r2.1
-      nvidia-scratch-space:
-        tags: _version.2_1
-
-  r3.0:
-    add_deps_recursive:
-      nvidia-inference-common-code:
-        version: r3.0
-      nvidia-scratch-space:
-        tags: _version.3_0
+  r2.1: {}
+  r3.0: {}
   r3.1:
-    add_deps_recursive:
-      nvidia-inference-common-code:
-        version: r3.1
-      nvidia-scratch-space:
-        tags: _version.4_0
     deps:
       - tags: install,pytorch,from.src,_for-nvidia-mlperf-inference-v3.1
         names:
@@ -259,11 +243,6 @@ versions:
       - tags: install,nccl,libs,_cuda
 
   r4.0:
-    add_deps_recursive:
-      nvidia-inference-common-code:
-        version: r4.0
-      nvidia-scratch-space:
-        tags: _version.4_0
     default_env:
       BUILD_TRTLLM: 1
     deps:
@@ -301,11 +280,6 @@ versions:
             - 8
 
   r4.1-dev:
-    add_deps_recursive:
-      nvidia-inference-common-code:
-        version: r4.0
-      nvidia-scratch-space:
-        tags: _version.4_1
     default_env:
       BUILD_TRTLLM: 1
     deps:
@@ -343,11 +317,6 @@ versions:
             - 8
 
   r4.1:
-    add_deps_recursive:
-      nvidia-inference-common-code:
-        version: r4.1
-      nvidia-scratch-space:
-        tags: _version.4_1
     default_env:
       BUILD_TRTLLM: 1
 

diff --git a/script/get-mlperf-inference-nvidia-common-code/meta.yaml b/script/get-mlperf-inference-nvidia-common-code/meta.yaml
@@ -9,7 +9,7 @@ deps:
 - inherit_variation_tags: true
   names:
   - mlperf-inference-results
-  tags: get,mlperf,inference,results,official,_code-only
+  tags: get,mlperf,inference,results,official
 new_env_keys:
 - +PYTHONPATH
 - MLC_MLPERF_INFERENCE_NVIDIA_CODE_PATH
@@ -37,19 +37,29 @@ versions:
     add_deps_recursive:
       mlperf-inference-results:
         version: v2.1
+        tags: _code-only
   r3.0:
     add_deps_recursive:
       mlperf-inference-results:
         version: v3.0
+        tags: _code-only
   r3.1:
     add_deps_recursive:
       mlperf-inference-results:
         version: v3.1
+        tags: _code-only
   r4.0:
     add_deps_recursive:
       mlperf-inference-results:
         version: v4.0
+        tags: _code-only
   r4.1:
     add_deps_recursive:
       mlperf-inference-results:
         version: v4.1
+        tags: _code-only
+  r4.1-dev:
+    add_deps_recursive:
+      mlperf-inference-results:
+        version: v4.0
+        tags: _code-only-for-v5.0
diff --git a/script/get-mlperf-inference-results/meta.yaml b/script/get-mlperf-inference-results/meta.yaml
@@ -38,6 +38,11 @@ variations:
       inference-results-repo:
         tags: _branch.mlc-code-only
     group: repo-branch
+  code-only-for-v5.0:
+    adr:
+      inference-results-repo:
+        tags: _branch.mlc-code-only-for-v5.0
+    group: repo-branch
   ctuning:
     env:
       GITHUB_REPO_OWNER: ctuning

diff --git a/script/push-mlperf-inference-results-to-github/run.sh b/script/push-mlperf-inference-results-to-github/run.sh
@@ -21,4 +21,6 @@ git commit -a -m "${MLC_MLPERF_RESULTS_REPO_COMMIT_MESSAGE}"
 echo ${MLC_GIT_PUSH_CMD}
 ${MLC_GIT_PUSH_CMD}
 
+test $? -eq 0 || (sleep $((RANDOM % 200 + 1)) && git pull && ${MLC_GIT_PUSH_CMD})
+
 test $? -eq 0 || exit $?