Merge from CTuning

cm-mlops/script/app-loadgen-generic-python/README-extra.md

@@ -2,7 +2,7 @@
 
 This portable CM script provides a unified API and CLI to benchmark ONNX models 
 using the [MLPerf loadgen](https://github.com/mlcommons/inference/tree/master/loadgen).
-It measures performance withou accuracy using randomly generated inputs. 
+It measures performance without accuracy using randomly generated inputs. 
 If you need accuracy too, please check [this portable CM script](../run-mlperf-inference-app).
 
 ## Development status

cm-mlops/script/app-loadgen-generic-python/_cm.yaml

@@ -140,6 +140,29 @@ variations:
     env:
       CM_MODEL: resnet50
 
+  custom:
+    group:
+      models
+    env:
+      CM_MODEL: custom
+
+  huggingface:
+    env:
+      CM_CUSTOM_MODEL_SOURCE: huggingface
+
+  custom,huggingface:
+    deps:
+    - tags: get,ml-model,huggingface
+      names:
+      - hf-downloader
+      update_tags_from_env_with_prefix:
+        "_model-stub.":
+        - CM_ML_MODEL_STUB
+
+  model-stub.#:
+    env:
+      CM_ML_MODEL_STUB: "#"
+
   retinanet:
     group:
       models

cm-mlops/script/app-loadgen-generic-python/src/ort.py

@@ -7,7 +7,7 @@
 from loadgen.model import Model, ModelFactory, ModelInput, ModelInputSampler
 
 ONNX_TO_NP_TYPE_MAP = {
-    "tensor(bool)": np.bool,
+    "tensor(bool)": bool,
     "tensor(int)": np.int32,
     "tensor(int32)": np.int32,
     "tensor(int8)": np.int8,

cm-mlops/script/get-ml-model-huggingface-zoo/_cm.json

@@ -40,6 +40,11 @@
         "CM_MODEL_ZOO_STUB": "#"
       }
     },
+    "onnx-subfolder": {
+      "env": {
+        "CM_HF_SUBFOLDER": "onnx"
+      }
+    },
     "pierreguillou_bert_base_cased_squad_v1.1_portuguese": {
       "env": {
         "CM_MODEL_ZOO_STUB": "pierreguillou/bert-base-cased-squad-v1.1-portuguese"

cm-mlops/script/get-ml-model-huggingface-zoo/download_model.py

@@ -21,6 +21,8 @@
         if model_filename == '': 
             model_filename = 'model.onnx'
 
+        subfolder = os.environ.get('CM_HF_SUBFOLDER', '')
+
         model_filenames = model_filename.split(',') if ',' in model_filename else [model_filename]
 
         # First must be model
@@ -35,10 +37,17 @@
             if extra_dir!='' and not os.path.exists(extra_dir):
                 os.makedirs(extra_dir)
 
-            hf_hub_download(repo_id=model_stub,
+            if subfolder == '':
+                 hf_hub_download(repo_id=model_stub,
+                            filename=model_filename,
+                            force_filename=model_filename,
+                            cache_dir=os.getcwd())
+            else:
+                 hf_hub_download(repo_id=model_stub,
+                            subfolder=subfolder,
                             filename=model_filename,
-                            cache_dir=os.getcwd(),
-                            force_filename=model_filename)
+                            force_filename=model_filename,
+                            cache_dir=os.getcwd())
 
         with open('tmp-run-env.out', 'w') as f:
             f.write(f"CM_ML_MODEL_FILE_WITH_PATH={os.path.join(os.getcwd(),base_model_filename)}")