sync v2.0 changes to main branch

docs/builder/build_kernel_latency_predictor.md

@@ -140,7 +140,7 @@ workspace = "/path/to/workspace/"
 builder_config.init(workspace)
 
 kernel_type = "conv-bn-relu"
-mark = "prior"
+mark = "test"
 backend = "tflite_cpu"
 error_threshold = 0.1
 
@@ -151,10 +151,10 @@ kernel_data = (cfgs_path, lats_path)
 
 # build latency predictor
 predictor, acc10, error_configs = build_predictor_by_data(
-    kernel_type, kernel_data, backend, error_threshold=error_threshold, mark="prior",
+    kernel_type, kernel_data, backend, error_threshold=error_threshold, mark=mark,
     save_path=os.path.join(workspace, "predictor_build", "results")
 )
-print(f'Iteration 0: acc10 {acc10}, error_configs number: {len(error_configs)}')    
+print(f'Iteration 0: acc10 {acc10}, error_configs number: {len(error_configs)}')
 ```
 
 In the implementation, the tuple of `kernel_data` includes items of `cfgs_path` and `lats_path`, indicating the config information and latency information respectively. `cfgs_path` and `lats_path` accept both json file path string or dictionary of models. In addition, if the config information and latency information are in the same data holder, users could directly specify `kernel_data = cfgs_path`.
@@ -174,7 +174,6 @@ from nn_meter.builder import build_initial_predictor_by_data
 workspace = "/path/to/workspace/"
 builder_config.init(workspace)
 
-backend = connect_backend(backend_name="tflite_cpu")
 kernel_type = "conv-bn-relu"
 backend = "tflite_cpu"
 error_threshold = 0.1
@@ -206,10 +205,9 @@ Here is an example for adaptive data sampling:
 from nn_meter.builder import builder_config
 from nn_meter.builder.backends import connect_backend
 from nn_meter.builder import build_adaptive_predictor_by_data
-workspace = "/data1/jiahang/working/release2test/"
+workspace = "/path/to/workspace/"
 builder_config.init(workspace)
 
-backend = connect_backend(backend_name="tflite_cpu")
 kernel_type = "conv-bn-relu"
 backend = "tflite_cpu"
 error_threshold = 0.1
@@ -221,14 +219,8 @@ predictor, data = build_adaptive_predictor_by_data(
 In the method `build_adaptive_predictor_by_data`, the parameter `kernel_data` indicates all training and testing data for current predictor training. The value of `kernel_data` could either be an instance of Dict generated by `build_initial_predictor_by_data` or `build_adaptive_predictor_by_data`, or be a instance of Tuple such as:
 
 ```python
-config_json_file = [
-    f'{workspace}/predictor_build/results/{kernel_type}_prior.json',
-    f'{workspace}/predictor_build/results/{kernel_type}_finegrained1.json',
-    f'{workspace}/predictor_build/results/{kernel_type}_finegrained2.json'
-]
-latency_json_file = [
-    f'{workspace}/predictor_build/results/profiled_{kernel_type}.json'
-]
+config_json_file = [f'{workspace}/predictor_build/results/{kernel_type}_prior.json'] # Add all needed json files name in the list
+latency_json_file = [f'{workspace}/predictor_build/results/profiled_{kernel_type}.json'] # Add all needed json files name in the list
 kernel_data = (config_json_file, latency_json_file)
 ```
 
@@ -252,7 +244,7 @@ predictor, data = build_predictor_for_kernel(
 )
 ```
 
-In the experiment of nn-Meter, we set `init_sample_num` as 1000, `finegrained_sample_num` as 10, `iteration` as 5, and `error_threshold` as 0.1.
+In the experiment of nn-Meter, we set default `init_sample_num` as 1000, `finegrained_sample_num` as 10, `iteration` as 5, and `error_threshold` as 0.1.
 
 nn-Meter also provided a end-to-end method for users to build a series of general latency predictors, named `nn_meter.builder.build_latency_predictor`. This method will build predictors for all kernels in `<workspace-path>/configs/predictorbuild_config.yaml` according to their corresponding parameters. The parameters includes `INIT_SAMPLE_NUM`, `FINEGRAINED_SAMPLE_NUM`, `ITERATION`, and `ERROR_THRESHOLD`. Here is an example:
 

docs/requirements/requirements_builder.txt

@@ -1,4 +1,5 @@
-tensorflow
+tensorflow==2.7.0
+# torch==1.10.0
 serial
 pure-python-adb
 typing

examples/nn-meter_builder_examples/build_kernel_latency_predictor.ipynb

@@ -28,7 +28,7 @@
    "source": [
     "# initialize builder config with workspace\n",
     "from nn_meter.builder import builder_config\n",
-    "workspace = \"/data1/jiahang/working/release2test/\"\n",
+    "workspace = \"/path/to/workspace/\"\n",
     "builder_config.init(workspace) "
    ]
   },

nn_meter/builder/backends/interface.py

@@ -83,7 +83,7 @@ def convert_model(self, model_path, save_path, input_shape=None):
             generated and used
         """
         # convert model and save the converted model to path `converted_model`
-        converted_model = ...
+        converted_model = model_path
         return converted_model
 
     def profile(self, converted_model, metrics = ['latency'], input_shape = None, **kwargs):

nn_meter/builder/kernel_predictor_builder/predictor_builder/utils.py

@@ -53,22 +53,40 @@ def get_flops_params(kernel_type, config):
 def collect_kernel_data(kernel_data, predict_label = 'latency'):
     if isinstance(kernel_data, dict):
         return kernel_data
+
     config, label = kernel_data
     if isinstance(config, list):
         config = collect_data(config)
+    else:
+        with open(config, 'r') as fp:
+            config = json.load(fp)
+
     if isinstance(label, list):
         label = collect_data(label)
-        if predict_label == 'latency':
-            from nn_meter.builder.backend_meta.utils import read_profiled_results
-            label = read_profiled_results(label)
-    return (config, label)
+    else:
+        with open(label, 'r') as fp:
+            label = json.load(fp)
+    if predict_label == 'latency':
+        from nn_meter.builder.backend_meta.utils import read_profiled_results
+        label = read_profiled_results(label)
+
+    for modules in config.keys():
+        for model_id in config[modules].keys():
+            try:
+                config[modules][model_id][predict_label] = label[modules][model_id][predict_label]
+            except:
+                pass
+
+    return config
 
 
 def collect_data(file_list):
+    file_list_copy = file_list[:]
+
     from ...utils import merge_info
-    data = file_list.pop(0)
+    data = file_list_copy.pop(0)
     with open(data, 'r') as fp:
         data = json.load(fp)
-    for file in file_list:
+    for file in file_list_copy:
         data = merge_info(new_info=file, prev_info=data)
     return data

nn_meter/builder/nn_meter_builder.py

@@ -68,7 +68,7 @@ def convert_models(backend, models, mode = 'predbuild', broken_point_mode = Fals
     return models
 
 
-def profile_models(backend, models, mode = 'ruletest', metrics = ["latency"], save_name = None,
+def profile_models(backend, models, mode = 'ruletest', metrics = ["latency"], save_name = "profiled_results.json",
                    have_converted = False, log_frequency = 50, broken_point_mode = False, **kwargs):
     """ run models with given backend and return latency of testcase models
 
@@ -231,25 +231,25 @@ def build_predictor_for_kernel(kernel_type, backend, init_sample_num = 1000, fin
 
 
 def build_initial_predictor_by_data(kernel_type, backend = None, init_sample_num = 20, error_threshold = 0.1, mark = '', predict_label = "latency"):
-    return build_predictor_for_kernel(kernel_type, backend, init_sample_num=init_sample_num, iteration=1, error_threshold=error_threshold, predict_label=predict_label, mark=f'{mark}')
+    return build_predictor_for_kernel(kernel_type, backend, init_sample_num=init_sample_num, iteration=1, error_threshold=error_threshold, predict_label=predict_label, mark=mark)
 
 
 def build_adaptive_predictor_by_data(kernel_type, kernel_data, backend = None, finegrained_sample_num = 20, error_threshold = 0.1, mark = '', predict_label = "latency"):
     """ Run adaptive sampler in one iteration based 
     """
     workspace_path = builder_config.get('WORKSPACE', 'predbuild')
     save_path = os.path.join(workspace_path, "results")
+    mark = 'finegrained' if mark == "" else f"finegrained_{mark}"
 
     from nn_meter.builder.kernel_predictor_builder import build_predictor_by_data, collect_kernel_data
     _, _, error_configs = build_predictor_by_data(kernel_type, kernel_data, backend = backend, error_threshold=error_threshold, save_path=None, predict_label=predict_label)
     new_kernel_data = sample_and_profile_kernel_data(kernel_type, finegrained_sample_num, backend,
                                                      sampling_mode='finegrained', configs=error_configs, mark=mark)
 
     # merge finegrained data with previous data and build new regression model
-    mark = mark if mark == "" else "_" + mark
     kernel_data = merge_info(new_info=new_kernel_data, prev_info=collect_kernel_data(kernel_data))
     predictor, acc10, error_configs = build_predictor_by_data(kernel_type, kernel_data, backend, error_threshold=error_threshold,
-                                                              mark=f'finegrained{mark}', save_path=save_path, predict_label=predict_label)
+                                                              mark=mark, save_path=save_path, predict_label=predict_label)
     logging.keyinfo(f'{mark}: acc10 {acc10}, error_configs number: {len(error_configs)}')
     return predictor, kernel_data
 

nn_meter/configs/builder/fusion_rule_tester/ruletest_config.yaml

@@ -1,4 +1,4 @@
-DETAIL: TRUE
+DETAIL: FALSE
 IMPLEMENT: tensorflow
 HW: 28
 CIN: 64

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='nn-meter',
-    version='1.1dev',
+    version='2.0',
     description='nn-Meter is a novel and efficient system to accurately predict the inference latency of DNN models on diverse edge devices.',
     long_description = open('README.md', encoding='utf-8').read(),
     long_description_content_type = 'text/markdown',

tests/unit_test/test_module_register.py

@@ -11,7 +11,11 @@ def register_by_meta(module_type, register_meta):
         yaml.dump(register_meta, fp)
     os.system(f"nn-meter register --{module_type} meta_file.yaml")
     os.remove("meta_file.yaml")
-    os.system(f"nn-meter unregister --{module_type} {register_meta['builtin_name']}")
+    if module_type in ["operator", "testcase"]:
+        implement = register_meta['implement']
+    else:
+        implement = ""
+    os.system(f"nn-meter unregister --{module_type} {register_meta['builtin_name']} {implement}")
 
 
 # test register backend

tests/unit_test/test_nn_modules.py

@@ -7,7 +7,7 @@
 
 if __name__ == '__main__':
     config = {
-        "HW": 1,
+        "HW": 24,
         "CIN": 144,
         "COUT": 32,
         "KERNEL_SIZE": 1, 
@@ -19,9 +19,12 @@
         "CIN3": 12,
         "CIN4": 12
     }
+    # test tersorflow kernels, tensorflow==2.7.0 or 2.6.0 is needed
     from nn_meter.builder.nn_modules.tf_networks import blocks
     for kernel in kernels:
         getattr(blocks, kernel)(config).test_block()
+
+    # test torch kernels, torch==1.10.0 or 1.9.0 is needed
     from nn_meter.builder.nn_modules.torch_networks import blocks
     for kernel in kernels:
         getattr(blocks, kernel)(config).test_block()