Merge from CTuning (#1046)

cm-mlops/script/calibrate-model-for.qaic/_cm.json

@@ -28,7 +28,7 @@
         "openimages-cal",
         "preprocessed-dataset"
       ],
-      "tags": "get,preprocessed,dataset,_calibration,openimages,_for.retinanet.onnx,_NCHW,_fp32"
+      "tags": "get,preprocessed,dataset,_calibration,openimages,_for.retinanet.onnx,_NCHW,_fp32,_custom-annotations"
     },
     {
       "enable_if_env":
@@ -39,7 +39,8 @@
       },
       "names": [
         "imagenet-cal",
-        "preprocessed-dataset"
+        "preprocessed-dataset",
+        "dataset-preprocessed"
       ],
       "tags": "get,dataset,imagenet,preprocessed,_calibration,_for.resnet50,_float32,_rgb32"
     },

cm-mlops/script/get-dataset-openimages/_cm.json

@@ -135,7 +135,7 @@
     "full": {
       "group": "size",
       "env": {
-        "CM_DATASET_SIZE": "24781"
+        "CM_DATASET_SIZE": ""
       }
     },
     "custom-annotations": {

cm-mlops/script/get-generic-python-lib/_cm.json

@@ -51,7 +51,9 @@
   "variations": {
     "package.#": {
       "env": {
-        "CM_GENERIC_PYTHON_PACKAGE_NAME": "#"
+        "CM_GENERIC_PYTHON_PACKAGE_NAME": "#",
+        "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "",
+        "CM_GENERIC_PYTHON_PIP_URL": ""
       }
     },
     "apache-tvm": {
@@ -363,8 +365,23 @@
         "pycocotools"
       ],
       "env": {
+        "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "pycocotools",
         "CM_GENERIC_PYTHON_PIP_URL": "pycocotools@git+https://github.com/NVIDIA/cocoapi#subdirectory=PythonAPI"
-      }
+      },
+      "deps": [
+        {
+          "names": [
+            "cython"
+          ],
+          "tags": "get,generic-python-lib,_package.cython"
+        },
+        {
+          "names": [
+            "numpy"
+          ],
+          "tags": "get,generic-python-lib,_package.numpy"
+        }
+      ]
     },
     "nvidia-pyindex": {
       "env": {

cm-mlops/script/get-generic-python-lib/customize.py

@@ -14,7 +14,12 @@ def preprocess(i):
     if package_name == '':
         return automation._available_variations({'meta':meta})
 
-    if env.get('CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS'):
+    extra = env.get('CM_GENERIC_PYTHON_PIP_EXTRA','')
+    if (pip_version and len(pip_version) > 1 and int(pip_version[0]) >= 23) and ('--break-system-packages' not in extra):
+        extra += '  --break-system-packages '
+        env['CM_PYTHON_PIP_COMMON_EXTRA'] = " --break-system-packages"
+
+    if env.get('CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS', '') != '':
         r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'uninstall_deps'})
         if r['return']>0: return r
 
@@ -39,11 +44,6 @@ def preprocess(i):
             if env.get(env_version_key,'')!='':
                 del(env[env_version_key])
 
-            extra = env.get('CM_GENERIC_PYTHON_PIP_EXTRA','')
-            if (pip_version and len(pip_version) > 1 and int(pip_version[0]) >= 23) and ('--break-system-packages' not in extra):
-                extra += '  --break-system-packages '
-                env['CM_PYTHON_PIP_COMMON_EXTRA'] = " --break-system-packages"
-
             # Check index URL
             index_url = env.get('CM_GENERIC_PYTHON_PIP_INDEX_URL','').strip()
             if index_url != '':

cm-mlops/script/get-generic-python-lib/uninstall_deps.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if [[ -n ${CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS} ]]; then
-    cmd="${CM_PYTHON_BIN_WITH_PATH} -m pip uninstall ${CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS} -y"
+    cmd="${CM_PYTHON_BIN_WITH_PATH} -m pip uninstall ${CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS} -y ${CM_PYTHON_PIP_COMMON_EXTRA}"
     echo "$cmd"
     eval "$cmd"
     test $? -eq 0 || exit $?

cm-mlops/script/get-mlperf-inference-src/_cm.json

@@ -89,6 +89,14 @@
         "patch"
       ]
     },
+    "openimages-nvidia-pycocotools": {
+      "env": {
+        "CM_GIT_PATCH_FILENAME": "openimages-pycocotools.patch"
+      },
+      "base": [
+        "patch"
+      ]
+    },
     "deepsparse": {
       "env": {
         "CM_GIT_URL": "https://github.com/neuralmagic/inference",

cm-mlops/script/get-mlperf-inference-src/patch/openimages-pycocotools.patch

@@ -0,0 +1,24 @@
+diff --git a/vision/classification_and_detection/tools/accuracy-openimages.py b/vision/classification_and_detection/tools/accuracy-openimages.py
+index 655ae5c..497dcbd 100644
+--- a/vision/classification_and_detection/tools/accuracy-openimages.py
++++ b/vision/classification_and_detection/tools/accuracy-openimages.py
+@@ -36,7 +36,7 @@ def main():
+     annotations_file = os.environ.get('DATASET_ANNOTATIONS_FILE_PATH')
+     if not annotations_file:
+         annotations_file = os.path.join(args.openimages_dir, "annotations/openimages-mlperf.json")
+-    cocoGt = COCO(annotations_file)
++    cocoGt = COCO(annotations_file, use_ext=True)
+
+     if args.use_inv_map:
+         inv_map = [0] + cocoGt.getCatIds() # First label in inv_map is not used
+@@ -100,8 +100,8 @@ def main():
+     with open(args.output_file, "w") as fp:
+         json.dump(detections, fp, sort_keys=True, indent=4)
+
+-    cocoDt = cocoGt.loadRes(args.output_file) # Load from file to bypass error with Python3
+-    cocoEval = COCOeval(cocoGt, cocoDt, iouType='bbox')
++    cocoDt = cocoGt.loadRes(args.output_file, use_ext=True) # Load from file to bypass error with Python3
++    cocoEval = COCOeval(cocoGt, cocoDt, iouType='bbox', use_ext=True)
+     cocoEval.params.imgIds = list(image_ids)
+     cocoEval.evaluate()
+     cocoEval.accumulate()

cm-mlops/script/get-preprocessed-dataset-generic/src/generic_preprocess.py

@@ -62,7 +62,7 @@ def center_crop(img):
 
 
 def preprocess_files(selected_filenames, source_dir, destination_dir, crop_percentage, square_side, inter_size, convert_to_bgr,
-    data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method):
+    data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, given_channel_stds, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method):
     "Go through the selected_filenames and preprocess all the files (optionally normalize and subtract mean)"
 
     output_filenames = []
@@ -93,6 +93,10 @@ def preprocess_files(selected_filenames, source_dir, destination_dir, crop_perce
             else:
                 image_data -= np.mean(image_data)
 
+        # Subtract standard deviations.
+        if len(given_channel_stds):
+            image_data /= given_channel_stds
+
         # NHWC -> NCHW.
         if data_layout == 'nchw':
             image_data = image_data[:,:,0:3].transpose(2, 0, 1)
@@ -151,6 +155,7 @@ def preprocess():
     normalize_data          = int(os.getenv('CM_DATASET_NORMALIZE_DATA', '0'))
     subtract_mean           = int(os.getenv('CM_DATASET_SUBTRACT_MEANS', '0'))
     given_channel_means     = os.getenv('CM_DATASET_GIVEN_CHANNEL_MEANS', '')
+    given_channel_stds     = os.getenv('CM_DATASET_GIVEN_CHANNEL_STDS', '')
     quant_scale             = float( os.environ['CM_DATASET_QUANT_SCALE'] )
     quant_offset            = float( os.environ['CM_DATASET_QUANT_OFFSET'] )
     quantize                = int( os.environ['CM_DATASET_QUANTIZE'] ) #1 for quantize to int8
@@ -161,12 +166,15 @@ def preprocess():
     if given_channel_means:
         given_channel_means = [ float(x) for x in given_channel_means.split(' ') ]
 
+    if given_channel_stds:
+        given_channel_stds = [ float(x) for x in given_channel_stds.split(' ') ]
+
     interpolation_method    = os.getenv('CM_DATASET_INTERPOLATION_METHOD', '')
 
     print(("From: {}, To: {}, Size: {}, Crop: {}, InterSize: {}, 2BGR: {}, OFF: {}, VOL: '{}', FOF: {},"+
         " DTYPE: {}, DLAYOUT: {}, EXT: {}, NORM: {}, SMEAN: {}, GCM: {}, QUANTIZE: {}, QUANT_SCALE: {}, QUANT_OFFSET: {}, CONV_UNSIGNED: {}, INTER: {}").format(
         source_dir, destination_dir, square_side, crop_percentage, inter_size, convert_to_bgr, offset, volume, fof_name,
-        data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method) )
+        data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, given_channel_stds, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method) )
 
     if interpolation_method == 'INTER_AREA':
         # Used for ResNet in pre_process_vgg.
@@ -197,7 +205,7 @@ def preprocess():
 
     output_filenames = preprocess_files(
         selected_filenames, source_dir, destination_dir, crop_percentage, square_side, inter_size, convert_to_bgr,
-        data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method)
+        data_type, data_layout, new_file_extension, normalize_data, subtract_mean, given_channel_means, given_channel_stds, quantize, quant_scale, quant_offset, convert_to_unsigned, interpolation_method)
 
     fof_full_path = os.path.join(destination_dir, fof_name)
     with open(fof_full_path, 'w') as fof:

cm-mlops/script/get-preprocessed-dataset-openimages/_cm.json

@@ -179,7 +179,7 @@
     },
     "custom-annotations": {
       "group": "annotations",
-      "add_deps_recursive": {
+      "ad": {
         "original-dataset": {
           "tags": "_custom-annotations"
         }
@@ -188,7 +188,7 @@
     "default-annotations": {
       "group": "annotations",
       "default": true,
-      "add_deps_recursive": {
+      "ad": {
         "original-dataset": {
           "tags": "_default-annotations"
         }
@@ -204,7 +204,11 @@
         "CM_DATASET_SUBTRACT_MEANS": "1",
         "CM_DATASET_GIVEN_CHANNEL_MEANS": "0.485 0.456 0.406",
         "CM_DATASET_GIVEN_CHANNEL_STDS": "0.229 0.224 0.225",
-        "CM_DATASET_NORMALIZE_DATA": "1"
+        "CM_DATASET_NORMALIZE_DATA": "0",
+        "CM_DATASET_NORMALIZE_LOWER": "0.0",
+        "CM_DATASET_NORMALIZE_UPPER": "1.0",
+        "CM_DATASET_CONVERT_TO_BGR": "0",
+        "CM_DATASET_CROP_FACTOR": "100.0"
       }
     },
     "for.retinanet.onnx,fp32": {

cm-mlops/script/get-preprocessed-dataset-openimages/preprocess.py

@@ -14,8 +14,10 @@
 preprocessed_dir = os.environ.get('CM_DATASET_PREPROCESSED_PATH', os.getcwd())
 
 if os.environ.get('CM_DATASET_REFERENCE_PREPROCESSOR', '1') == "0":
-    import generic_preprocess
-    generic_preprocess.preprocess()
+    #import generic_preprocess
+    #generic_preprocess.preprocess()
+    import preprocess_image_dataset as pp
+    pp.preprocess()
 else:
     dataset_list = os.environ.get('CM_DATASET_ANNOTATIONS_FILE_PATH', None)
     img_format = os.environ.get('CM_DATASET_DATA_LAYOUT', 'NHWC')

cm-mlops/script/process-mlperf-accuracy/_cm.json

@@ -142,16 +142,17 @@
     "openimages": {
       "deps": [
         {
-          "names": [
-            "openimages-dataset",
-            "openimages-original"
-          ],
-          "tags": "get,dataset,openimages,original"
+          "tags": "get,dataset-aux,openimages,annotations"
         },
         {
-          "tags": "get,generic-python-lib,_pycocotools"
+          "tags": "get,generic-python-lib,_nvidia-pycocotools"
         }
       ],
+      "ad": {
+        "inference-src": {
+          "tags": "_openimages-nvidia-pycocotools"
+        }
+      },
       "env": {
         "CM_DATASET": "openimages"
       },

cm-mlops/script/process-mlperf-accuracy/customize.py

@@ -25,7 +25,8 @@ def preprocess(i):
 
         if dataset == "openimages":
             env['DATASET_ANNOTATIONS_FILE_PATH'] = env['CM_DATASET_ANNOTATIONS_FILE_PATH']
-            dataset_dir = os.path.join(env['CM_DATASET_PATH'], "..", "..")
+            #dataset_dir = os.path.join(env['CM_DATASET_PATH'], "..", "..")
+            dataset_dir = os.getcwd() # not used, just to keep the script happy
             CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", \
                 "accuracy-openimages.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, \
                     "mlperf_log_accuracy.json") + "' --openimages-dir '" + dataset_dir + "' --verbose > '" + \

cm-mlops/script/reproduce-mlperf-inference-qualcomm/README_aws_dl2q.24xlarge.md

@@ -71,7 +71,7 @@ cm run script --tags=generate-run-cmds,inference,_performance-only --device=qaic
 ```
 cm run script --tags=generate-run-cmds,inference,_performance-only --device=qaic --backend=glow \
 --scenario=Offline  --implementation=kilt --model=resnet50 \
---test_query_count=400000 --precision=uint8 --rerun --adr.compiler.tags=gcc \
+--precision=uint8 --rerun --adr.compiler.tags=gcc \
 --adr.mlperf-inference-implementation.tags=_bs.8,_dl2q.24xlarge --execution-mode=valid --quiet
 ```
 Expected performance is ~157500
@@ -82,7 +82,7 @@ Expected performance is ~157500
 ```
 cm run script --tags=generate-run-cmds,inference,_accuracy-only --device=qaic --backend=glow \
 --scenario=Offline  --implementation=kilt --model=resnet50 \
---test_query_count=400000 --precision=uint8 --rerun --adr.compiler.tags=gcc \
+--precision=uint8 --rerun --adr.compiler.tags=gcc \
 --adr.mlperf-inference-implementation.tags=_bs.8,_dl2q.24xlarge --execution-mode=valid --quiet
 ```
 
@@ -93,4 +93,34 @@ Expected accuracy is 75.936%
 
 ## RetinaNet
 
-*TODO*
+### Quick performance run
+
+```
+cm run script --tags=generate-run-cmds,inference,_performance-only --device=qaic --backend=glow \
+--scenario=Offline  --implementation=kilt --model=retinanet --test_query_count=40000 --precision=uint8 \
+--rerun --quiet --adr.mlperf-inference-implementation.tags=_loadgen-batch-size.1,_dl2q.24xlarge,_bs.1 \
+--adr.compiler.tags=gcc --execution-mode=test 
+```
+
+### Full valid run
+
+```
+cm run script --tags=generate-run-cmds,inference,_performance-only --device=qaic --backend=glow \
+--scenario=Offline  --implementation=kilt --model=retinanet \
+--precision=uint8 --rerun --adr.compiler.tags=gcc --adr.dataset-preprocessed.tags=_custom-annotations\
+--adr.mlperf-inference-implementation.tags=_bs.1,_dl2q.24xlarge --execution-mode=valid --quiet
+```
+Expected performance is ~2200
+* Use `--scenario=Server --server_target_qps=2050` to run the server scenario
+
+### Accuracy run
+
+```
+cm run script --tags=generate-run-cmds,inference,_accuracy-only --device=qaic --backend=glow \
+--scenario=Offline  --implementation=kilt --model=retinanet \
+--precision=uint8 --rerun --adr.compiler.tags=gcc --adr.dataset-preprocessed.tags=_custom-annotations \
+--adr.mlperf-inference-implementation.tags=_bs.1,_dl2q.24xlarge --execution-mode=valid --quiet
+```
+
+The expected accuracy is 37.xx
+

cm-mlops/script/reproduce-mlperf-inference-qualcomm/_cm.yaml

@@ -160,7 +160,7 @@ deps:
     names:
       - openimages-preprocessed
       - dataset-preprocessed
-    tags: get,dataset,preprocessed,openimages,_for.retinanet.onnx,_NCHW,_validation,_full
+    tags: get,dataset,preprocessed,openimages,_for.retinanet.onnx,_NCHW,_validation,_custom-annotations
 
 
 
@@ -330,7 +330,7 @@ variations:
       CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://zenodo.org/record/6617981/files/resnext50_32x4d_fpn.pth"
       kilt_model_name: retinanet
       kilt_input_count: 1
-      kilt_model_disable_nms: ''
+      #kilt_model_disable_nms: ''
       kilt_model_max_detections: 600
       kilt_output_count: 1
       kilt_input_format: "FLOAT32,-1,3,800,800"
@@ -528,6 +528,19 @@ variations:
   dl2q.24xlarge,bert-99,offline:
     env:
       qaic_activation_count: "14"
+
   dl2q.24xlarge,retinanet,offline:
     env:
       qaic_activation_count: "14"
+
+  dl2q.24xlarge,resnet50,server:
+    env:
+      qaic_activation_count: "3"
+
+  dl2q.24xlarge,bert-99,server:
+    env:
+      qaic_activation_count: "14"
+
+  dl2q.24xlarge,retinanet,server:
+    env:
+      qaic_activation_count: "14"