regular CMX improvements for MLPerf (#1382)

.github/workflows/cla.yml

@@ -4,10 +4,11 @@ on:
   issue_comment:
     types: [created]
   pull_request_target:
-    types: [opened,closed,synchronize]
+    types: [opened,closed,synchronize]  
 
 jobs:
   cla-check:
+    if: github.repository_owner == 'mlcommons'
     runs-on: ubuntu-latest
     steps:
       - name: "MLCommons CLA bot check"
@@ -22,7 +23,7 @@ jobs:
           path-to-signatures: 'cla-bot/v1/cla.json'
           # branch should not be protected
           branch: 'main'
-          allowlist: user1,bot*
+          allowlist: user1,mlcommons-bot,bot*
           remote-organization-name: mlcommons
           remote-repository-name: systems
 

.github/workflows/test-cmx-image-classification-onnx.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: 'CMX: image classification with ONNX'
+name: 'Image classification with ONNX via CMX'
 
 on:
   pull_request:
@@ -28,8 +28,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install cmind
-        cmx pull repo --url=https://github.com/mlcommons/cm4mlops
+        python3 -m pip install cmx4mlops
         cmx test core
     - name: Test image classification with ONNX
       run: |

.github/workflows/test-cmx-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml

@@ -0,0 +1,66 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MLPerf inference bert (deepsparse, tf, onnxruntime, pytorch)
+
+on:
+  pull_request_target:
+    branches: [ "main", "master", "dev"]
+    paths:
+      - '.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.12 didn't work on 20240305 - need to check
+        python-version: [ "3.11" ]
+        backend: [ "deepsparse", "tf", "onnxruntime", "pytorch" ]
+        precision: [ "int8", "fp32" ]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+          - backend: tf
+          - backend: pytorch
+          - backend: onnxruntime
+          - precision: fp32
+          - os: windows-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Configure git longpaths (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        git config --system core.longpaths true
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmx4mlops
+    - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os == 'windows-latest'
+      run: |
+        cmx run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --precision=${{ matrix.precision }} --target_qps=1 --v --quiet
+    - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os != 'windows-latest'
+      run: |
+        cmx run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 --v --quiet
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
+      if: github.repository_owner == 'ctuning'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "[email protected]"
+          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        cmx run script --tags=push,github,mlperf,inference,submission  --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions"  --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-cmx-mlperf-inference-resnet50.yml

@@ -0,0 +1,66 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: 'MLPerf inference ResNet50 via CMX'
+
+on:
+  pull_request_target:
+    branches: [ "main", "master", "dev"]
+    paths:
+      - '.github/workflows/test-cmx-mlperf-inference-resnet50.yml'
+      - '**'
+      - '!**.md'
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    env:
+      CM_INDEX: "on"
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [ "3.12" ]
+        backend: [ "onnxruntime", "tf" ]
+        implementation: [ "python", "cpp" ]
+        exclude:
+          - backend: tf
+            implementation: cpp
+          - os: macos-latest
+            backend: tf
+          - os: windows-latest
+            implementation: cpp
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Configure git longpaths (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        git config --system core.longpaths true
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmx4mlops
+    - name: Test MLPerf Inference ResNet50 (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        cmx run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1  --quiet 
+    - name: Test MLPerf Inference ResNet50 (Linux/macOS)
+      if: matrix.os != 'windows-latest'
+      run: |
+        cmx run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 --v --quiet 
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
+      if: github.repository_owner == 'ctuning'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "[email protected]"
+          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        cmx run script --tags=push,github,mlperf,inference,submission  --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions"  --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-cmx-mlperf-inference-rgat.yml

@@ -0,0 +1,51 @@
+name: MLPerf inference R-GAT via CMX
+
+on:
+  pull_request:
+    branches: [ "main", "master", "dev" ]
+    paths:
+      - '.github/workflows/test-mlperf-inference-rgat.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [ "3.12" ]
+        backend: [ "pytorch" ]
+        implementation: [ "python" ]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Configure git longpaths (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        git config --system core.longpaths true
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmx4mlops
+    - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }}
+      run: |
+        cmx run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --adr.inference-src.tags=_branch.dev --pull_changes=yes --pull_inference_changes=yes  --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet  --v --target_qps=1
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
+      if: github.repository_owner == 'ctuning'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "[email protected]"
+          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        cmx run script --tags=push,github,mlperf,inference,submission  --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions"  --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-cmx.yml

@@ -0,0 +1,52 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: CMX test
+
+on:
+  pull_request:
+    branches: [ "master", "main", "dev" ]
+    paths:
+      - '.github/workflows/test-cm.yml'
+      - 'cm/**'
+      - '!cm/**.md'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        on: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+          - python-version: "3.7"
+            on: "macos-latest"
+    runs-on: "${{ matrix.on }}"
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies and test cm pull repo
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        pip install -r requirements.txt
+        python -m pip install --ignore-installed --verbose pip setuptools
+        python -m pip install cmx4mlops
+        python -m cmind
+        cm init
+    - name: Test
+      run: |
+        python tests/test_cm.py
+        cd && mkdir tmp_cm_repo
+        cd tmp_cm_repo && cm init repo
+        cmx add script tmp_cm_repo:my-test-script --tags=test,script
+        cmx add script .:my-test-script2 --tags=test2,script
+        cd && cmx add repo my-test-repo
+        cmx add script my-test-repo:my-test-script --tags=test,script
+        cd $HOME/CM/repos/my-test-repo && cmx add script .:my-test-script2 --tags=test2,script
+

HISTORY.CM.md

@@ -1,4 +1,4 @@
-This document narrates the history of the creation and design of CM and CM4MLOps (also known as CK2) 
+This document narrates the history of the creation and design of CM, CM4MLOps and MLPerf automations (also known as CK2) 
 by [Grigori Fursin](https://cKnowledge.org/gfursin). It also highlights the donation of this open-source technology to MLCommons, 
 aimed at benefiting the broader community and fostering its ongoing development as a collaborative, community-driven initiative:
 

README.md

@@ -11,14 +11,14 @@
 
 ## Collective Knowledge
 
-[Collective Knowledge (CK, CM, CM4MLOps, CM4MLPerf and CMX)](https://cKnowledge.org) 
+[Collective Knowledge (CK)](https://cKnowledge.org) 
 is an educational community project to learn how to run AI, ML and other emerging workloads 
 in the most efficient and cost-effective way across diverse models, data sets, software and hardware:
 [ [white paper](https://arxiv.org/abs/2406.16791) ].
 
 It includes the following sub-projects.
 
-### Collective Minds (CM)
+### Collective Mind (CM)
 
 The Collective Mind (CM) project, or Collective Minds, facilitates the
 decomposition of complex software systems into portable, reusable, and
@@ -140,6 +140,11 @@ Copyright (c) 2014-2021 cTuning foundation
 
 * [Grigori Fursin](https://cKnowledge.org/gfursin) (FlexAI, cTuning)
 
+## Maintainers
+
+* CM, CM4MLOps and MLPerf automations: MLCommons
+* CMX (the next generation of CM): Grigori Fursin
+
 ## Long-term vision
 
 To learn more about the motivation behind CK and CM technology, please explore the following presentations:

cm/cmind/__init__.py

@@ -9,7 +9,7 @@
 # White paper: https://arxiv.org/abs/2406.16791
 # Project contributors: https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md
 
-__version__ = "3.5.3"
+__version__ = "3.5.3.1"
 
 from cmind.core import access
 from cmind.core import x

cm/cmind/core.py

@@ -1125,6 +1125,7 @@ def _x(self, i, control):
                    print ('  -raise - raise Python error when automation action fails')
                    print ('  -time - print elapsed time for a given automation')
                    print ('  -profile - profile a given automation')
+                   print ('  -i - print info about available memory and disk space')
                    print ('  -repro - record various info to the cmx-repro directory to replay CMX command')
                    print ('')
                    print ('Check https://github.com/mlcommons/ck/tree/master/cm/docs/cmx for more details.')

cmx4mlops/README.md

@@ -0,0 +1 @@
+TBD

cmx4mlops/cmx4mlops/repo/.gitignore

@@ -0,0 +1,20 @@
+build
+build/*
+MANIFEST
+*.pyc
+__pycache__
+develop-eggs/
+dist/
+eggs/
+.eggs/
+lib/
+lib64/
+sdist/
+wheels/
+.cache/
+.coverage
+htmlcov
+*tmp/
+*tmp-ck-*/
+local/cache/
+