diff --git a/cm-mlops/script/calibrate-model-for.qaic/_cm.json b/cm-mlops/script/calibrate-model-for.qaic/_cm.json index 764f9239f0..79a22a8a06 100644 --- a/cm-mlops/script/calibrate-model-for.qaic/_cm.json +++ b/cm-mlops/script/calibrate-model-for.qaic/_cm.json @@ -28,7 +28,7 @@ "openimages-cal", "preprocessed-dataset" ], - "tags": "get,preprocessed,dataset,_calibration,openimages,_for.retinanet.onnx,_NCHW,_fp32" + "tags": "get,preprocessed,dataset,_calibration,openimages,_for.retinanet.onnx,_NCHW,_fp32,_custom-annotations" }, { "enable_if_env": @@ -39,7 +39,8 @@ }, "names": [ "imagenet-cal", - "preprocessed-dataset" + "preprocessed-dataset", + "dataset-preprocessed" ], "tags": "get,dataset,imagenet,preprocessed,_calibration,_for.resnet50,_float32,_rgb32" }, diff --git a/cm-mlops/script/get-dataset-openimages/_cm.json b/cm-mlops/script/get-dataset-openimages/_cm.json index 53ff5278a8..0d835e6405 100644 --- a/cm-mlops/script/get-dataset-openimages/_cm.json +++ b/cm-mlops/script/get-dataset-openimages/_cm.json @@ -135,7 +135,7 @@ "full": { "group": "size", "env": { - "CM_DATASET_SIZE": "24781" + "CM_DATASET_SIZE": "" } }, "custom-annotations": { diff --git a/cm-mlops/script/get-generic-python-lib/_cm.json b/cm-mlops/script/get-generic-python-lib/_cm.json index da273c399b..8664092a7a 100644 --- a/cm-mlops/script/get-generic-python-lib/_cm.json +++ b/cm-mlops/script/get-generic-python-lib/_cm.json @@ -51,7 +51,9 @@ "variations": { "package.#": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "#" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "#", + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "", + "CM_GENERIC_PYTHON_PIP_URL": "" } }, "apache-tvm": { @@ -363,8 +365,23 @@ "pycocotools" ], "env": { + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "pycocotools", "CM_GENERIC_PYTHON_PIP_URL": "pycocotools@git+https://github.com/NVIDIA/cocoapi#subdirectory=PythonAPI" - } + }, + "deps": [ + { + "names": [ + "cython" + ], + "tags": "get,generic-python-lib,_package.cython" + }, + { + "names": [ + "numpy" + ], + "tags": "get,generic-python-lib,_package.numpy" + } + ] }, "nvidia-pyindex": { "env": { diff --git a/cm-mlops/script/get-generic-python-lib/customize.py b/cm-mlops/script/get-generic-python-lib/customize.py index 495ca5f0f7..80568cce3e 100644 --- a/cm-mlops/script/get-generic-python-lib/customize.py +++ b/cm-mlops/script/get-generic-python-lib/customize.py @@ -14,7 +14,12 @@ def preprocess(i): if package_name == '': return automation._available_variations({'meta':meta}) - if env.get('CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS'): + extra = env.get('CM_GENERIC_PYTHON_PIP_EXTRA','') + if (pip_version and len(pip_version) > 1 and int(pip_version[0]) >= 23) and ('--break-system-packages' not in extra): + extra += ' --break-system-packages ' + env['CM_PYTHON_PIP_COMMON_EXTRA'] = " --break-system-packages" + + if env.get('CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS', '') != '': r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'uninstall_deps'}) if r['return']>0: return r @@ -39,11 +44,6 @@ def preprocess(i): if env.get(env_version_key,'')!='': del(env[env_version_key]) - extra = env.get('CM_GENERIC_PYTHON_PIP_EXTRA','') - if (pip_version and len(pip_version) > 1 and int(pip_version[0]) >= 23) and ('--break-system-packages' not in extra): - extra += ' --break-system-packages ' - env['CM_PYTHON_PIP_COMMON_EXTRA'] = " --break-system-packages" - # Check index URL index_url = env.get('CM_GENERIC_PYTHON_PIP_INDEX_URL','').strip() if index_url != '': diff --git a/cm-mlops/script/get-generic-python-lib/uninstall_deps.sh b/cm-mlops/script/get-generic-python-lib/uninstall_deps.sh index 09cde493dc..eeddf36d7b 100644 --- a/cm-mlops/script/get-generic-python-lib/uninstall_deps.sh +++ b/cm-mlops/script/get-generic-python-lib/uninstall_deps.sh @@ -1,7 +1,7 @@ #!/bin/bash if [[ -n ${CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS} ]]; then - cmd="${CM_PYTHON_BIN_WITH_PATH} -m pip uninstall ${CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS} -y" + cmd="${CM_PYTHON_BIN_WITH_PATH} -m pip uninstall ${CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS} -y ${CM_PYTHON_PIP_COMMON_EXTRA}" echo "$cmd" eval "$cmd" test $? -eq 0 || exit $? diff --git a/cm-mlops/script/get-mlperf-inference-src/_cm.json b/cm-mlops/script/get-mlperf-inference-src/_cm.json index fad0d5a6c0..b842e9277a 100644 --- a/cm-mlops/script/get-mlperf-inference-src/_cm.json +++ b/cm-mlops/script/get-mlperf-inference-src/_cm.json @@ -89,6 +89,14 @@ "patch" ] }, + "openimages-nvidia-pycocotools": { + "env": { + "CM_GIT_PATCH_FILENAME": "openimages-pycocotools.patch" + }, + "base": [ + "patch" + ] + }, "deepsparse": { "env": { "CM_GIT_URL": "https://github.com/neuralmagic/inference", diff --git a/cm-mlops/script/get-mlperf-inference-src/patch/openimages-pycocotools.patch b/cm-mlops/script/get-mlperf-inference-src/patch/openimages-pycocotools.patch new file mode 100644 index 0000000000..7dc3126781 --- /dev/null +++ b/cm-mlops/script/get-mlperf-inference-src/patch/openimages-pycocotools.patch @@ -0,0 +1,24 @@ +diff --git a/vision/classification_and_detection/tools/accuracy-openimages.py b/vision/classification_and_detection/tools/accuracy-openimages.py +index 655ae5c..497dcbd 100644 +--- a/vision/classification_and_detection/tools/accuracy-openimages.py ++++ b/vision/classification_and_detection/tools/accuracy-openimages.py +@@ -36,7 +36,7 @@ def main(): + annotations_file = os.environ.get('DATASET_ANNOTATIONS_FILE_PATH') + if not annotations_file: + annotations_file = os.path.join(args.openimages_dir, "annotations/openimages-mlperf.json") +- cocoGt = COCO(annotations_file) ++ cocoGt = COCO(annotations_file, use_ext=True) + + if args.use_inv_map: + inv_map = [0] + cocoGt.getCatIds() # First label in inv_map is not used +@@ -100,8 +100,8 @@ def main(): + with open(args.output_file, "w") as fp: + json.dump(detections, fp, sort_keys=True, indent=4) + +- cocoDt = cocoGt.loadRes(args.output_file) # Load from file to bypass error with Python3 +- cocoEval = COCOeval(cocoGt, cocoDt, iouType='bbox') ++ cocoDt = cocoGt.loadRes(args.output_file, use_ext=True) # Load from file to bypass error with Python3 ++ cocoEval = COCOeval(cocoGt, cocoDt, iouType='bbox', use_ext=True) + cocoEval.params.imgIds = list(image_ids) + cocoEval.evaluate() + cocoEval.accumulate() diff --git a/cm-mlops/script/get-preprocessed-dataset-generic/src/generic_preprocess.py b/cm-mlops/script/get-preprocessed-dataset-generic/src/generic_preprocess.py index ef22422d39..c21e71c2e5 100644 --- a/cm-mlops/script/get-preprocessed-dataset-generic/src/generic_preprocess.py +++ b/cm-mlops/script/get-preprocessed-dataset-generic/src/generic_preprocess.py @@ -62,7 +62,7 @@ def center_crop(img): def preprocess_files(selected_filenames, source_dir, destination_dir, crop_percentage, square_side, inter_size, convert_to_bgr, - data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method): + data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, given_channel_stds, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method): "Go through the selected_filenames and preprocess all the files (optionally normalize and subtract mean)" output_filenames = [] @@ -93,6 +93,10 @@ def preprocess_files(selected_filenames, source_dir, destination_dir, crop_perce else: image_data -= np.mean(image_data) + # Subtract standard deviations. + if len(given_channel_stds): + image_data /= given_channel_stds + # NHWC -> NCHW. if data_layout == 'nchw': image_data = image_data[:,:,0:3].transpose(2, 0, 1) @@ -151,6 +155,7 @@ def preprocess(): normalize_data = int(os.getenv('CM_DATASET_NORMALIZE_DATA', '0')) subtract_mean = int(os.getenv('CM_DATASET_SUBTRACT_MEANS', '0')) given_channel_means = os.getenv('CM_DATASET_GIVEN_CHANNEL_MEANS', '') + given_channel_stds = os.getenv('CM_DATASET_GIVEN_CHANNEL_STDS', '') quant_scale = float( os.environ['CM_DATASET_QUANT_SCALE'] ) quant_offset = float( os.environ['CM_DATASET_QUANT_OFFSET'] ) quantize = int( os.environ['CM_DATASET_QUANTIZE'] ) #1 for quantize to int8 @@ -161,12 +166,15 @@ def preprocess(): if given_channel_means: given_channel_means = [ float(x) for x in given_channel_means.split(' ') ] + if given_channel_stds: + given_channel_stds = [ float(x) for x in given_channel_stds.split(' ') ] + interpolation_method = os.getenv('CM_DATASET_INTERPOLATION_METHOD', '') print(("From: {}, To: {}, Size: {}, Crop: {}, InterSize: {}, 2BGR: {}, OFF: {}, VOL: '{}', FOF: {},"+ " DTYPE: {}, DLAYOUT: {}, EXT: {}, NORM: {}, SMEAN: {}, GCM: {}, QUANTIZE: {}, QUANT_SCALE: {}, QUANT_OFFSET: {}, CONV_UNSIGNED: {}, INTER: {}").format( source_dir, destination_dir, square_side, crop_percentage, inter_size, convert_to_bgr, offset, volume, fof_name, - data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method) ) + data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, given_channel_stds, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method) ) if interpolation_method == 'INTER_AREA': # Used for ResNet in pre_process_vgg. @@ -197,7 +205,7 @@ def preprocess(): output_filenames = preprocess_files( selected_filenames, source_dir, destination_dir, crop_percentage, square_side, inter_size, convert_to_bgr, - data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method) + data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, given_channel_stds, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method) fof_full_path = os.path.join(destination_dir, fof_name) with open(fof_full_path, 'w') as fof: diff --git a/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json b/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json index d1692574ad..bce5f211ac 100644 --- a/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json +++ b/cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json @@ -179,7 +179,7 @@ }, "custom-annotations": { "group": "annotations", - "add_deps_recursive": { + "ad": { "original-dataset": { "tags": "_custom-annotations" } @@ -188,7 +188,7 @@ "default-annotations": { "group": "annotations", "default": true, - "add_deps_recursive": { + "ad": { "original-dataset": { "tags": "_default-annotations" } @@ -204,7 +204,11 @@ "CM_DATASET_SUBTRACT_MEANS": "1", "CM_DATASET_GIVEN_CHANNEL_MEANS": "0.485 0.456 0.406", "CM_DATASET_GIVEN_CHANNEL_STDS": "0.229 0.224 0.225", - "CM_DATASET_NORMALIZE_DATA": "1" + "CM_DATASET_NORMALIZE_DATA": "0", + "CM_DATASET_NORMALIZE_LOWER": "0.0", + "CM_DATASET_NORMALIZE_UPPER": "1.0", + "CM_DATASET_CONVERT_TO_BGR": "0", + "CM_DATASET_CROP_FACTOR": "100.0" } }, "for.retinanet.onnx,fp32": { diff --git a/cm-mlops/script/get-preprocessed-dataset-openimages/preprocess.py b/cm-mlops/script/get-preprocessed-dataset-openimages/preprocess.py index bc5b2db406..33aee9d6f4 100644 --- a/cm-mlops/script/get-preprocessed-dataset-openimages/preprocess.py +++ b/cm-mlops/script/get-preprocessed-dataset-openimages/preprocess.py @@ -14,8 +14,10 @@ preprocessed_dir = os.environ.get('CM_DATASET_PREPROCESSED_PATH', os.getcwd()) if os.environ.get('CM_DATASET_REFERENCE_PREPROCESSOR', '1') == "0": - import generic_preprocess - generic_preprocess.preprocess() + #import generic_preprocess + #generic_preprocess.preprocess() + import preprocess_image_dataset as pp + pp.preprocess() else: dataset_list = os.environ.get('CM_DATASET_ANNOTATIONS_FILE_PATH', None) img_format = os.environ.get('CM_DATASET_DATA_LAYOUT', 'NHWC') diff --git a/cm-mlops/script/process-mlperf-accuracy/_cm.json b/cm-mlops/script/process-mlperf-accuracy/_cm.json index 13e1ce27f9..98ff5d9228 100644 --- a/cm-mlops/script/process-mlperf-accuracy/_cm.json +++ b/cm-mlops/script/process-mlperf-accuracy/_cm.json @@ -142,16 +142,17 @@ "openimages": { "deps": [ { - "names": [ - "openimages-dataset", - "openimages-original" - ], - "tags": "get,dataset,openimages,original" + "tags": "get,dataset-aux,openimages,annotations" }, { - "tags": "get,generic-python-lib,_pycocotools" + "tags": "get,generic-python-lib,_nvidia-pycocotools" } ], + "ad": { + "inference-src": { + "tags": "_openimages-nvidia-pycocotools" + } + }, "env": { "CM_DATASET": "openimages" }, diff --git a/cm-mlops/script/process-mlperf-accuracy/customize.py b/cm-mlops/script/process-mlperf-accuracy/customize.py index eaeea9eb26..a9ed1279fa 100644 --- a/cm-mlops/script/process-mlperf-accuracy/customize.py +++ b/cm-mlops/script/process-mlperf-accuracy/customize.py @@ -25,7 +25,8 @@ def preprocess(i): if dataset == "openimages": env['DATASET_ANNOTATIONS_FILE_PATH'] = env['CM_DATASET_ANNOTATIONS_FILE_PATH'] - dataset_dir = os.path.join(env['CM_DATASET_PATH'], "..", "..") + #dataset_dir = os.path.join(env['CM_DATASET_PATH'], "..", "..") + dataset_dir = os.getcwd() # not used, just to keep the script happy CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", \ "accuracy-openimages.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, \ "mlperf_log_accuracy.json") + "' --openimages-dir '" + dataset_dir + "' --verbose > '" + \ diff --git a/cm-mlops/script/reproduce-mlperf-inference-qualcomm/README_aws_dl2q.24xlarge.md b/cm-mlops/script/reproduce-mlperf-inference-qualcomm/README_aws_dl2q.24xlarge.md index 0b74589441..8cf35760aa 100644 --- a/cm-mlops/script/reproduce-mlperf-inference-qualcomm/README_aws_dl2q.24xlarge.md +++ b/cm-mlops/script/reproduce-mlperf-inference-qualcomm/README_aws_dl2q.24xlarge.md @@ -71,7 +71,7 @@ cm run script --tags=generate-run-cmds,inference,_performance-only --device=qaic ``` cm run script --tags=generate-run-cmds,inference,_performance-only --device=qaic --backend=glow \ --scenario=Offline --implementation=kilt --model=resnet50 \ ---test_query_count=400000 --precision=uint8 --rerun --adr.compiler.tags=gcc \ +--precision=uint8 --rerun --adr.compiler.tags=gcc \ --adr.mlperf-inference-implementation.tags=_bs.8,_dl2q.24xlarge --execution-mode=valid --quiet ``` Expected performance is ~157500 @@ -82,7 +82,7 @@ Expected performance is ~157500 ``` cm run script --tags=generate-run-cmds,inference,_accuracy-only --device=qaic --backend=glow \ --scenario=Offline --implementation=kilt --model=resnet50 \ ---test_query_count=400000 --precision=uint8 --rerun --adr.compiler.tags=gcc \ +--precision=uint8 --rerun --adr.compiler.tags=gcc \ --adr.mlperf-inference-implementation.tags=_bs.8,_dl2q.24xlarge --execution-mode=valid --quiet ``` @@ -93,4 +93,34 @@ Expected accuracy is 75.936% ## RetinaNet -*TODO* +### Quick performance run + +``` +cm run script --tags=generate-run-cmds,inference,_performance-only --device=qaic --backend=glow \ +--scenario=Offline --implementation=kilt --model=retinanet --test_query_count=40000 --precision=uint8 \ +--rerun --quiet --adr.mlperf-inference-implementation.tags=_loadgen-batch-size.1,_dl2q.24xlarge,_bs.1 \ +--adr.compiler.tags=gcc --execution-mode=test +``` + +### Full valid run + +``` +cm run script --tags=generate-run-cmds,inference,_performance-only --device=qaic --backend=glow \ +--scenario=Offline --implementation=kilt --model=retinanet \ +--precision=uint8 --rerun --adr.compiler.tags=gcc --adr.dataset-preprocessed.tags=_custom-annotations\ +--adr.mlperf-inference-implementation.tags=_bs.1,_dl2q.24xlarge --execution-mode=valid --quiet +``` +Expected performance is ~2200 +* Use `--scenario=Server --server_target_qps=2050` to run the server scenario + +### Accuracy run + +``` +cm run script --tags=generate-run-cmds,inference,_accuracy-only --device=qaic --backend=glow \ +--scenario=Offline --implementation=kilt --model=retinanet \ +--precision=uint8 --rerun --adr.compiler.tags=gcc --adr.dataset-preprocessed.tags=_custom-annotations \ +--adr.mlperf-inference-implementation.tags=_bs.1,_dl2q.24xlarge --execution-mode=valid --quiet +``` + +The expected accuracy is 37.xx + diff --git a/cm-mlops/script/reproduce-mlperf-inference-qualcomm/_cm.yaml b/cm-mlops/script/reproduce-mlperf-inference-qualcomm/_cm.yaml index 9b5b354995..99d336e2fa 100644 --- a/cm-mlops/script/reproduce-mlperf-inference-qualcomm/_cm.yaml +++ b/cm-mlops/script/reproduce-mlperf-inference-qualcomm/_cm.yaml @@ -160,7 +160,7 @@ deps: names: - openimages-preprocessed - dataset-preprocessed - tags: get,dataset,preprocessed,openimages,_for.retinanet.onnx,_NCHW,_validation,_full + tags: get,dataset,preprocessed,openimages,_for.retinanet.onnx,_NCHW,_validation,_custom-annotations @@ -330,7 +330,7 @@ variations: CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://zenodo.org/record/6617981/files/resnext50_32x4d_fpn.pth" kilt_model_name: retinanet kilt_input_count: 1 - kilt_model_disable_nms: '' + #kilt_model_disable_nms: '' kilt_model_max_detections: 600 kilt_output_count: 1 kilt_input_format: "FLOAT32,-1,3,800,800" @@ -528,6 +528,19 @@ variations: dl2q.24xlarge,bert-99,offline: env: qaic_activation_count: "14" + dl2q.24xlarge,retinanet,offline: env: qaic_activation_count: "14" + + dl2q.24xlarge,resnet50,server: + env: + qaic_activation_count: "3" + + dl2q.24xlarge,bert-99,server: + env: + qaic_activation_count: "14" + + dl2q.24xlarge,retinanet,server: + env: + qaic_activation_count: "14"