From cbac69d8926fdf7247ac9640c9c35ac3173fb285 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 15 Nov 2023 16:25:27 +0100 Subject: [PATCH 1/3] fixing typo --- cm-mlops/script/app-loadgen-generic-python/README-extra.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cm-mlops/script/app-loadgen-generic-python/README-extra.md b/cm-mlops/script/app-loadgen-generic-python/README-extra.md index c4d1ba3e4f..b9e5373761 100644 --- a/cm-mlops/script/app-loadgen-generic-python/README-extra.md +++ b/cm-mlops/script/app-loadgen-generic-python/README-extra.md @@ -2,7 +2,7 @@ This portable CM script provides a unified API and CLI to benchmark ONNX models using the [MLPerf loadgen](https://github.com/mlcommons/inference/tree/master/loadgen). -It measures performance withou accuracy using randomly generated inputs. +It measures performance without accuracy using randomly generated inputs. If you need accuracy too, please check [this portable CM script](../run-mlperf-inference-app). ## Development status From 45c33f072e62d7b01fd003c67938d2ed26e1dbcc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 16 Nov 2023 17:06:58 +0000 Subject: [PATCH 2/3] Support custom huggingface model in app-loadgen-generic --- .../app-loadgen-generic-python/_cm.yaml | 21 +++++++++++++++++++ .../app-loadgen-generic-python/src/ort.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cm-mlops/script/app-loadgen-generic-python/_cm.yaml b/cm-mlops/script/app-loadgen-generic-python/_cm.yaml index 27157b85bf..fd4349617e 100644 --- a/cm-mlops/script/app-loadgen-generic-python/_cm.yaml +++ b/cm-mlops/script/app-loadgen-generic-python/_cm.yaml @@ -140,6 +140,27 @@ variations: env: CM_MODEL: resnet50 + custom: + group: + models + env: + CM_MODEL: custom + + huggingface: + env: + CM_CUSTOM_MODEL_SOURCE: huggingface + + custom,huggingface: + deps: + - tags: get,ml-model,huggingface + update_tags_from_env_with_prefix: + "_model-stub.": + - CM_ML_MODEL_STUB + + model-stub.#: + env: + CM_ML_MODEL_STUB: "#" + retinanet: group: models diff --git a/cm-mlops/script/app-loadgen-generic-python/src/ort.py b/cm-mlops/script/app-loadgen-generic-python/src/ort.py index 320c5dacdc..488b3392a6 100644 --- a/cm-mlops/script/app-loadgen-generic-python/src/ort.py +++ b/cm-mlops/script/app-loadgen-generic-python/src/ort.py @@ -7,7 +7,7 @@ from loadgen.model import Model, ModelFactory, ModelInput, ModelInputSampler ONNX_TO_NP_TYPE_MAP = { - "tensor(bool)": np.bool, + "tensor(bool)": bool, "tensor(int)": np.int32, "tensor(int32)": np.int32, "tensor(int8)": np.int8, From a02c4ca9f109bffd44e8f2e0ac797831d6335449 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 16 Nov 2023 17:49:12 +0000 Subject: [PATCH 3/3] Support onnx subfolder for hf download --- .../script/app-loadgen-generic-python/_cm.yaml | 2 ++ .../script/get-ml-model-huggingface-zoo/_cm.json | 5 +++++ .../download_model.py | 15 ++++++++++++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cm-mlops/script/app-loadgen-generic-python/_cm.yaml b/cm-mlops/script/app-loadgen-generic-python/_cm.yaml index fd4349617e..eb04f3a7dd 100644 --- a/cm-mlops/script/app-loadgen-generic-python/_cm.yaml +++ b/cm-mlops/script/app-loadgen-generic-python/_cm.yaml @@ -153,6 +153,8 @@ variations: custom,huggingface: deps: - tags: get,ml-model,huggingface + names: + - hf-downloader update_tags_from_env_with_prefix: "_model-stub.": - CM_ML_MODEL_STUB diff --git a/cm-mlops/script/get-ml-model-huggingface-zoo/_cm.json b/cm-mlops/script/get-ml-model-huggingface-zoo/_cm.json index 374625a57f..ed918c8d72 100644 --- a/cm-mlops/script/get-ml-model-huggingface-zoo/_cm.json +++ b/cm-mlops/script/get-ml-model-huggingface-zoo/_cm.json @@ -40,6 +40,11 @@ "CM_MODEL_ZOO_STUB": "#" } }, + "onnx-subfolder": { + "env": { + "CM_HF_SUBFOLDER": "onnx" + } + }, "pierreguillou_bert_base_cased_squad_v1.1_portuguese": { "env": { "CM_MODEL_ZOO_STUB": "pierreguillou/bert-base-cased-squad-v1.1-portuguese" diff --git a/cm-mlops/script/get-ml-model-huggingface-zoo/download_model.py b/cm-mlops/script/get-ml-model-huggingface-zoo/download_model.py index 5ed130a501..df81c2c9a8 100644 --- a/cm-mlops/script/get-ml-model-huggingface-zoo/download_model.py +++ b/cm-mlops/script/get-ml-model-huggingface-zoo/download_model.py @@ -21,6 +21,8 @@ if model_filename == '': model_filename = 'model.onnx' + subfolder = os.environ.get('CM_HF_SUBFOLDER', '') + model_filenames = model_filename.split(',') if ',' in model_filename else [model_filename] # First must be model @@ -35,10 +37,17 @@ if extra_dir!='' and not os.path.exists(extra_dir): os.makedirs(extra_dir) - hf_hub_download(repo_id=model_stub, + if subfolder == '': + hf_hub_download(repo_id=model_stub, + filename=model_filename, + force_filename=model_filename, + cache_dir=os.getcwd()) + else: + hf_hub_download(repo_id=model_stub, + subfolder=subfolder, filename=model_filename, - cache_dir=os.getcwd(), - force_filename=model_filename) + force_filename=model_filename, + cache_dir=os.getcwd()) with open('tmp-run-env.out', 'w') as f: f.write(f"CM_ML_MODEL_FILE_WITH_PATH={os.path.join(os.getcwd(),base_model_filename)}")