Merge from CTuning

cm-mlops/automation/script/module.py

@@ -4133,12 +4133,14 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
                            print (r['string'])
                            print ("")
 
-            note = '''^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+            note = '''
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Note that it may be a portability issue of a third-party tool or a native script 
-wrapped and unified by this portable CM script. In such case, please report this issue 
-with a full log at "https://github.com/mlcommons/ck". The CM concept is to collaboratively 
-fix such issues inside portable CM scripts to make existing tools and native scripts
-more portable, interoperable and deterministic. Thank you'''
+wrapped and unified by this automation recipe (CM script). In such case, 
+please report this issue with a full log at "https://github.com/mlcommons/ck". 
+The CM concept is to collaboratively fix such issues inside portable CM scripts 
+to make existing tools and native scripts more portable, interoperable 
+and deterministic. Thank you!'''
 
             return {'return':2, 'error':'Portable CM script failed (name = {}, return code = {})\n\n{}'.format(meta['alias'], rc, note)}
 

cm-mlops/automation/script/module_misc.py

@@ -1187,9 +1187,10 @@ def regenerate_script_cmd(i):
 
 
     skip_input_for_fake_run = docker_settings.get('skip_input_for_fake_run', [])
+    add_quotes_to_keys = docker_settings.get('add_quotes_to_keys', [])
 
 
-    def rebuild_flags(i_run_cmd, fake_run, skip_input_for_fake_run, key_prefix):
+    def rebuild_flags(i_run_cmd, fake_run, skip_input_for_fake_run, add_quotes_to_keys, key_prefix):
 
         run_cmd = ''
 
@@ -1212,16 +1213,22 @@ def rebuild_flags(i_run_cmd, fake_run, skip_input_for_fake_run, key_prefix):
 
             v = i_run_cmd[k]
 
+            q = '\\"' if long_key in add_quotes_to_keys else ''
+
             if type(v)==dict:
-                run_cmd += rebuild_flags(v, fake_run, skip_input_for_fake_run, long_key)
+                run_cmd += rebuild_flags(v, fake_run, skip_input_for_fake_run, add_quotes_to_keys, long_key)
             elif type(v)==list:
-                run_cmd+=' --'+long_key+',='+','.join(v)
+                x = ''
+                for vv in v:
+                    if x != '': x+=','
+                    x+=q+str(vv)+q
+                run_cmd+=' --'+long_key+',=' + x
             else:
-                run_cmd+=' --'+long_key+'='+str(v)
+                run_cmd+=' --'+long_key+'='+q+str(v)+q
 
         return run_cmd    
 
-    run_cmd += rebuild_flags(i_run_cmd, fake_run, skip_input_for_fake_run, '')
+    run_cmd += rebuild_flags(i_run_cmd, fake_run, skip_input_for_fake_run, add_quotes_to_keys, '')
 
     run_cmd = docker_run_cmd_prefix + ' && ' + run_cmd if docker_run_cmd_prefix!='' else run_cmd
 

cm-mlops/script/app-image-corner-detection/_cm.json

@@ -3,6 +3,10 @@
   "automation_alias": "script",
   "automation_uid": "5b4e0237da074764",
   "category": "Modular application pipeline",
+  "deps": [
+    {"tags":"detect,os"},
+    {"tags":"detect,cpu"}
+  ],
   "posthook_deps": [
     {
       "skip_if_env": {

cm-mlops/script/app-loadgen-generic-python/README-extra.md

@@ -221,12 +221,9 @@ cmr "python app loadgen-generic _onnxruntime _cuda _custom _huggingface _model-s
 cmr "python app loadgen-generic _onnxruntime _cuda _custom _huggingface _model-stub.Intel/gpt-j-6B-int8-static" --adr.hf-downloader.model_filename=model.onnx --adr.hf-downloader.full_subfolder=. --samples=2
 ```
 
-
-cmr "python app loadgen-generic _onnxruntime _custom _huggingface _model-stub.runwayml/stable-diffusion-v1-5" --adr.hf-downloader.model_filename=onnx/unet/model.onnx,onnx/unet/weights.pb --samples=2
-
-
-TBD: some cases that are not yet fully supported (data types, etc):
+TBD: some cases that are not yet fully supported (data types, input mismatch, etc):
 ```bash
+cmr "python app loadgen-generic _onnxruntime _custom _huggingface _model-stub.runwayml/stable-diffusion-v1-5" --adr.hf-downloader.revision=onnx --adr.hf-downloader.model_filename=unet/model.onnx,unet/weights.pb --samples=2
 cmr "python app loadgen-generic _onnxruntime _cuda _custom _huggingface _model-stub.microsoft/Mistral-7B-v0.1-onnx" --adr.hf-downloader.model_filename=Mistral-7B-v0.1.onnx,Mistral-7B-v0.1.onnx.data  --samples=2
 cmr "python app loadgen-generic _onnxruntime _cuda _custom _huggingface _model-stub.alpindale/Llama-2-7b-ONNX" --adr.hf-downloader.model_filename=FP16/LlamaV2_7B_float16.onnx --adr.hf-downloader.full_subfolder=FP16 --samples=2
 ```
@@ -269,7 +266,7 @@ Available flags mapped to environment variables:
 ## Running this app via Docker
 
 ```bash
-cm docker script "python app loadgen-generic _onnxruntime _cuda _custom _huggingface _model-stub.steerapi/Llama-2-7b-chat-hf-onnx-awq-w8" --adr.hf-downloader.model_filename=onnx/decoder_model_merged_quantized.onnx,onnx/decoder_model_merged_quantized.onnx_data --samples=2 --output_dir=. --docker_cm_repo=ctuning@mlcommons-ck
+cm docker script "python app loadgen-generic _onnxruntime _custom _huggingface _model-stub.ctuning/mlperf-inference-bert-onnx-fp32-squad-v1.1" --adr.hf-downloader.model_filename=model.onnx  --samples=2 --output_dir=new_results --docker_cm_repo=ctuning@mlcommons-ck
 ```
 
 ## Tuning CPU performance via CM experiment

cm-mlops/script/app-loadgen-generic-python/run.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
+
 ${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/src/main.py ${CM_RUN_OPTS} ${CM_ML_MODEL_FILE_WITH_PATH}
 test $? -eq 0 || exit 1

cm-mlops/script/app-mlperf-inference-cpp/customize.py

@@ -6,9 +6,16 @@ def preprocess(i):
 
     os_info = i['os_info']
 
-#    if os_info['platform'] == 'windows':
-#        return {'return':1, 'error': 'Windows is not supported in this script yet'}
+    automation = i['automation']
+
+    meta = i['meta']
 
+#    if os_info['platform'] == 'windows':
+#        # Currently support only LLVM on Windows
+#        print ('# Forcing LLVM on Windows')        
+#        r = automation.update_deps({'deps':meta['post_deps'], 'update_deps':{'compile-program': {'adr':{'compiler':{'tags':'llvm'}}}}})
+#        if r['return']>0: return r
+
     env = i['env']
 
     if env.get('CM_MLPERF_SKIP_RUN', '') == "yes":

cm-mlops/script/app-mlperf-inference-cpp/tests/win.bat

@@ -0,0 +1,8 @@
+rem TBD: current not compiling - need to check ...
+
+cmr "install llvm prebuilt" --version=16.0.4
+cmr "install llvm prebuilt" --version=17.0.6
+
+cmr "get lib onnxruntime lang-cpp _cpu" --version=1.11.1
+cmr "get lib onnxruntime lang-cpp _cpu" --version=1.13.1
+cmr "get lib onnxruntime lang-cpp _cpu" --version=1.15.1

cm-mlops/script/app-mlperf-inference/customize.py

@@ -203,7 +203,7 @@ def postprocess(i):
         if os.path.exists(env['CM_MLPERF_USER_CONF']):
             shutil.copy(env['CM_MLPERF_USER_CONF'], 'user.conf')
 
-        result = mlperf_utils.get_result_from_log(env['CM_MLPERF_LAST_RELEASE'], model, scenario, output_dir, mode)
+        result, valid = mlperf_utils.get_result_from_log(env['CM_MLPERF_LAST_RELEASE'], model, scenario, output_dir, mode)
         power = None
         power_efficiency = None
         if mode == "performance":
@@ -221,8 +221,11 @@ def postprocess(i):
         if not state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model].get(scenario):
             state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario] = {}
         state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario][mode] = result
+        state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario][mode+'_valid'] = valid[mode]
+
         if power:
             state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario]['power'] = power
+            state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario]['power_valid'] = valid['power']
         if power_efficiency:
             state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario]['power_efficiency'] = power_efficiency
 
@@ -281,7 +284,6 @@ def postprocess(i):
 
                 for xd in xdirs:
                     xpath = os.path.join(cache.path, xd)
-                    print (xpath)
                     if os.path.isdir(xpath):
                         r = cm.access({'action':'system', 'automation':'utils', 'path':xpath, 'cmd':'git rev-parse HEAD'})
                         if r['return'] == 0 and r['ret'] == 0:
@@ -459,6 +461,7 @@ def postprocess(i):
     if env.get('CM_DUMP_SYSTEM_INFO', True):
         dump_script_output("detect,os", env, state, 'new_env', os.path.join(output_dir, "os_info.json"))
         dump_script_output("detect,cpu", env, state, 'new_env', os.path.join(output_dir, "cpu_info.json"))
+        env['CM_DUMP_RAW_PIP_FREEZE_FILE_PATH'] = os.path.join(env['CM_MLPERF_OUTPUT_DIR'], "pip_freeze.raw")
         dump_script_output("dump,pip,freeze", env, state, 'new_state', os.path.join(output_dir, "pip_freeze.json"))
 
     return {'return':0}

cm-mlops/script/app-stable-diffusion-onnx-py/README-extra.md

@@ -0,0 +1,30 @@
+# Examples
+
+CM interface for https://huggingface.co/runwayml/stable-diffusion-v1-5/tree/onnx
+
+```bash
+cm run script "install python-venv" --name=sd-test
+cm run script "get generic-python-lib _package.optimum[onnxruntime]" --adr.python.name=sd-test
+cm run script "activate python-venv" --name=sd-test
+
+cm run script "python app stable-diffusion onnx" --adr.python.name=sd-test --text="crazy programmer"
+
+cm rm cache -f
+cm run script "python app stable-diffusion onnx _cuda" --adr.python.name=sd-test --text="crazy programmer"
+
+cm docker script "python app stable-diffusion onnx" --text="crazy programmer" --output=. --docker_cm_repo=ctuning@mlcommons-ck --env.CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO=xyz4
+
+```
+
+
+
+# Resources
+
+* https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+* https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/tree/main
+* https://huggingface.co/CompVis/stable-diffusion-v1-4/tree/main
+* https://huggingface.co/runwayml/stable-diffusion-v1-5
+* https://huggingface.co/bes-dev/stable-diffusion-v1-4-onnx
+* https://onnxruntime.ai/docs/tutorials/csharp/stable-diffusion-csharp.html
+* https://huggingface.co/runwayml/stable-diffusion-v1-5/tree/main
+* https://huggingface.co/docs/optimum/onnxruntime/usage_guides/models

cm-mlops/script/app-stable-diffusion-onnx-py/_cm.yaml

@@ -0,0 +1,106 @@
+alias: app-stable-diffusion-onnx-py
+uid: 4d33981ac3534b3b
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+category: "Modular AI/ML application pipeline"
+
+tags:
+- app
+- stable
+- diffusion
+- stable-diffusion
+- onnx
+- python
+
+
+deps:
+- tags: detect,os
+- tags: get,sys-utils-cm
+- names:
+  - python
+  - python3
+  tags: get,python3
+
+- tags: get,cuda
+  names:
+  - cuda
+  enable_if_env:
+    USE_CUDA:
+    - yes
+- tags: get,cudnn
+  names:
+  - cudnn
+  enable_if_env:
+    USE_CUDA:
+    - yes
+
+
+
+
+
+
+
+- tags: get,generic-python-lib,_package.optimum[onnxruntime]
+  names:
+  - optimum
+  skip_if_env:
+    USE_CUDA:
+    - yes
+
+- tags: get,generic-python-lib,_package.optimum[onnxruntime-gpu]
+  names:
+  - optimum
+  enable_if_env:
+    USE_CUDA:
+    - yes
+
+- tags: get,generic-python-lib,_package.diffusers
+  names:
+  - diffusers
+
+
+- tags: get,ml-model,huggingface,zoo,_model-stub.runwayml/stable-diffusion-v1-5
+  revision: onnx
+  model_filename: model_index.json
+  full_subfolder: .
+
+
+variations:
+  cuda:
+    group: target
+    env:
+      USE_CUDA: yes
+      CM_DEVICE: cuda:0
+
+  cpu:
+    group: target
+    default: yes
+    env:
+      USE_CPU: yes
+      CM_DEVICE: cpu
+
+input_mapping:
+  text: CM_APP_STABLE_DIFFUSION_ONNX_PY_TEXT
+  output: CM_APP_STABLE_DIFFUSION_ONNX_PY_OUTPUT
+
+
+input_description:
+  text: 
+    desc: "Text to generate image"
+  output: 
+    desc: "Output directory"
+
+
+docker:
+  skip_run_cmd: 'no'
+  all_gpus: 'yes'
+  input_paths:
+    - output
+  add_quotes_to_keys:
+    - text
+  skip_input_for_fake_run:
+    - text
+    - output
+    - env.CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO

cm-mlops/script/app-stable-diffusion-onnx-py/process.py

@@ -0,0 +1,34 @@
+# https://huggingface.co/runwayml/stable-diffusion-v1-5/tree/onnx
+
+import os
+
+from optimum.onnxruntime import ORTStableDiffusionPipeline
+
+output = os.environ.get('CM_APP_STABLE_DIFFUSION_ONNX_PY_OUTPUT','')
+
+f = os.path.join(output, 'output.png')
+
+if os.path.isfile(f):
+    os.remove(f)
+
+cm_model_path = os.environ.get('CM_ML_MODEL_PATH','')
+if cm_model_path == '':
+    print ('Error: CM_ML_MODEL_PATH env is not defined')
+    exit(1)
+
+device = os.environ.get('CM_DEVICE','')
+
+pipeline = ORTStableDiffusionPipeline.from_pretrained(cm_model_path, local_files_only=True).to(device)
+
+text = os.environ.get('CM_APP_STABLE_DIFFUSION_ONNX_PY_TEXT','')
+if text == '': text = "a photo of an astronaut riding a horse on mars"
+
+
+print ('')
+print ('Generating imaged based on "{}"'.format(text))
+
+image = pipeline(text).images[0]
+
+image.save(f)
+
+print ('Image recorded to "{}"'.format(f))

cm-mlops/script/app-stable-diffusion-onnx-py/run.bat

@@ -0,0 +1,2 @@
+%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\process.py
+IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%

cm-mlops/script/app-stable-diffusion-onnx-py/run.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+${CM_PYTHON_BIN} ${CM_TMP_CURRENT_SCRIPT_PATH}/process.py
+test $? -eq 0 || exit 1