From 0b5e0ce0bc4758bb1b0bfbdd233a83853d6ce908 Mon Sep 17 00:00:00 2001 From: jiahangxu Date: Sun, 26 Jun 2022 23:34:56 -0400 Subject: [PATCH 1/3] fix little bugs in v2.0 --- nn_meter/builder/nn_meter_builder.py | 2 +- .../configs/builder/fusion_rule_tester/ruletest_config.yaml | 2 +- setup.py | 2 +- tests/unit_test/test_module_register.py | 6 +++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nn_meter/builder/nn_meter_builder.py b/nn_meter/builder/nn_meter_builder.py index 181e269a..303ec0d9 100644 --- a/nn_meter/builder/nn_meter_builder.py +++ b/nn_meter/builder/nn_meter_builder.py @@ -68,7 +68,7 @@ def convert_models(backend, models, mode = 'predbuild', broken_point_mode = Fals return models -def profile_models(backend, models, mode = 'ruletest', metrics = ["latency"], save_name = None, +def profile_models(backend, models, mode = 'ruletest', metrics = ["latency"], save_name = "profiled_results.json", have_converted = False, log_frequency = 50, broken_point_mode = False, **kwargs): """ run models with given backend and return latency of testcase models diff --git a/nn_meter/configs/builder/fusion_rule_tester/ruletest_config.yaml b/nn_meter/configs/builder/fusion_rule_tester/ruletest_config.yaml index 1fe66a5e..fa41c411 100644 --- a/nn_meter/configs/builder/fusion_rule_tester/ruletest_config.yaml +++ b/nn_meter/configs/builder/fusion_rule_tester/ruletest_config.yaml @@ -1,4 +1,4 @@ -DETAIL: TRUE +DETAIL: FALSE IMPLEMENT: tensorflow HW: 28 CIN: 64 diff --git a/setup.py b/setup.py index b5802ee9..296ae736 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='nn-meter', - version='1.1dev', + version='2.0a1', description='nn-Meter is a novel and efficient system to accurately predict the inference latency of DNN models on diverse edge devices.', long_description = open('README.md', encoding='utf-8').read(), long_description_content_type = 'text/markdown', diff --git a/tests/unit_test/test_module_register.py b/tests/unit_test/test_module_register.py index 7bb863c5..a3a7b6ff 100644 --- a/tests/unit_test/test_module_register.py +++ b/tests/unit_test/test_module_register.py @@ -11,7 +11,11 @@ def register_by_meta(module_type, register_meta): yaml.dump(register_meta, fp) os.system(f"nn-meter register --{module_type} meta_file.yaml") os.remove("meta_file.yaml") - os.system(f"nn-meter unregister --{module_type} {register_meta['builtin_name']}") + if module_type in ["operator", "testcase"]: + implement = register_meta['implement'] + else: + implement = "" + os.system(f"nn-meter unregister --{module_type} {register_meta['builtin_name']} {implement}") # test register backend From e66d7ef3b9e9afec7c62f346e7f039dab634bfc8 Mon Sep 17 00:00:00 2001 From: jiahangxu Date: Mon, 27 Jun 2022 05:00:25 -0400 Subject: [PATCH 2/3] fix bugs in v2.0 --- .../builder/build_kernel_latency_predictor.md | 20 ++++--------- docs/requirements/requirements_builder.txt | 3 +- .../build_kernel_latency_predictor.ipynb | 2 +- nn_meter/builder/backends/interface.py | 2 +- .../predictor_builder/utils.py | 30 +++++++++++++++---- nn_meter/builder/nn_meter_builder.py | 6 ++-- tests/unit_test/test_nn_modules.py | 5 +++- 7 files changed, 41 insertions(+), 27 deletions(-) diff --git a/docs/builder/build_kernel_latency_predictor.md b/docs/builder/build_kernel_latency_predictor.md index 91971e92..2f0dfc4e 100644 --- a/docs/builder/build_kernel_latency_predictor.md +++ b/docs/builder/build_kernel_latency_predictor.md @@ -140,7 +140,7 @@ workspace = "/path/to/workspace/" builder_config.init(workspace) kernel_type = "conv-bn-relu" -mark = "prior" +mark = "test" backend = "tflite_cpu" error_threshold = 0.1 @@ -151,10 +151,10 @@ kernel_data = (cfgs_path, lats_path) # build latency predictor predictor, acc10, error_configs = build_predictor_by_data( - kernel_type, kernel_data, backend, error_threshold=error_threshold, mark="prior", + kernel_type, kernel_data, backend, error_threshold=error_threshold, mark=mark, save_path=os.path.join(workspace, "predictor_build", "results") ) -print(f'Iteration 0: acc10 {acc10}, error_configs number: {len(error_configs)}') +print(f'Iteration 0: acc10 {acc10}, error_configs number: {len(error_configs)}') ``` In the implementation, the tuple of `kernel_data` includes items of `cfgs_path` and `lats_path`, indicating the config information and latency information respectively. `cfgs_path` and `lats_path` accept both json file path string or dictionary of models. In addition, if the config information and latency information are in the same data holder, users could directly specify `kernel_data = cfgs_path`. @@ -174,7 +174,6 @@ from nn_meter.builder import build_initial_predictor_by_data workspace = "/path/to/workspace/" builder_config.init(workspace) -backend = connect_backend(backend_name="tflite_cpu") kernel_type = "conv-bn-relu" backend = "tflite_cpu" error_threshold = 0.1 @@ -206,10 +205,9 @@ Here is an example for adaptive data sampling: from nn_meter.builder import builder_config from nn_meter.builder.backends import connect_backend from nn_meter.builder import build_adaptive_predictor_by_data -workspace = "/data1/jiahang/working/release2test/" +workspace = "/path/to/workspace/" builder_config.init(workspace) -backend = connect_backend(backend_name="tflite_cpu") kernel_type = "conv-bn-relu" backend = "tflite_cpu" error_threshold = 0.1 @@ -221,14 +219,8 @@ predictor, data = build_adaptive_predictor_by_data( In the method `build_adaptive_predictor_by_data`, the parameter `kernel_data` indicates all training and testing data for current predictor training. The value of `kernel_data` could either be an instance of Dict generated by `build_initial_predictor_by_data` or `build_adaptive_predictor_by_data`, or be a instance of Tuple such as: ```python -config_json_file = [ - f'{workspace}/predictor_build/results/{kernel_type}_prior.json', - f'{workspace}/predictor_build/results/{kernel_type}_finegrained1.json', - f'{workspace}/predictor_build/results/{kernel_type}_finegrained2.json' -] -latency_json_file = [ - f'{workspace}/predictor_build/results/profiled_{kernel_type}.json' -] +config_json_file = [f'{workspace}/predictor_build/results/{kernel_type}_prior.json'] # Add all needed json files name in the list +latency_json_file = [f'{workspace}/predictor_build/results/profiled_{kernel_type}.json'] # Add all needed json files name in the list kernel_data = (config_json_file, latency_json_file) ``` diff --git a/docs/requirements/requirements_builder.txt b/docs/requirements/requirements_builder.txt index 507fb8d3..a1e93a40 100644 --- a/docs/requirements/requirements_builder.txt +++ b/docs/requirements/requirements_builder.txt @@ -1,4 +1,5 @@ -tensorflow +tensorflow==2.7.0 +# torch==1.10.0 serial pure-python-adb typing diff --git a/examples/nn-meter_builder_examples/build_kernel_latency_predictor.ipynb b/examples/nn-meter_builder_examples/build_kernel_latency_predictor.ipynb index 747b6b96..e1bac291 100644 --- a/examples/nn-meter_builder_examples/build_kernel_latency_predictor.ipynb +++ b/examples/nn-meter_builder_examples/build_kernel_latency_predictor.ipynb @@ -28,7 +28,7 @@ "source": [ "# initialize builder config with workspace\n", "from nn_meter.builder import builder_config\n", - "workspace = \"/data1/jiahang/working/release2test/\"\n", + "workspace = \"/path/to/workspace/\"\n", "builder_config.init(workspace) " ] }, diff --git a/nn_meter/builder/backends/interface.py b/nn_meter/builder/backends/interface.py index 0da3b6f5..6c5a5d55 100644 --- a/nn_meter/builder/backends/interface.py +++ b/nn_meter/builder/backends/interface.py @@ -83,7 +83,7 @@ def convert_model(self, model_path, save_path, input_shape=None): generated and used """ # convert model and save the converted model to path `converted_model` - converted_model = ... + converted_model = model_path return converted_model def profile(self, converted_model, metrics = ['latency'], input_shape = None, **kwargs): diff --git a/nn_meter/builder/kernel_predictor_builder/predictor_builder/utils.py b/nn_meter/builder/kernel_predictor_builder/predictor_builder/utils.py index 0d7c730b..43f2b849 100644 --- a/nn_meter/builder/kernel_predictor_builder/predictor_builder/utils.py +++ b/nn_meter/builder/kernel_predictor_builder/predictor_builder/utils.py @@ -53,22 +53,40 @@ def get_flops_params(kernel_type, config): def collect_kernel_data(kernel_data, predict_label = 'latency'): if isinstance(kernel_data, dict): return kernel_data + config, label = kernel_data if isinstance(config, list): config = collect_data(config) + else: + with open(config, 'r') as fp: + config = json.load(fp) + if isinstance(label, list): label = collect_data(label) - if predict_label == 'latency': - from nn_meter.builder.backend_meta.utils import read_profiled_results - label = read_profiled_results(label) - return (config, label) + else: + with open(label, 'r') as fp: + label = json.load(fp) + if predict_label == 'latency': + from nn_meter.builder.backend_meta.utils import read_profiled_results + label = read_profiled_results(label) + + for modules in config.keys(): + for model_id in config[modules].keys(): + try: + config[modules][model_id][predict_label] = label[modules][model_id][predict_label] + except: + pass + + return config def collect_data(file_list): + file_list_copy = file_list[:] + from ...utils import merge_info - data = file_list.pop(0) + data = file_list_copy.pop(0) with open(data, 'r') as fp: data = json.load(fp) - for file in file_list: + for file in file_list_copy: data = merge_info(new_info=file, prev_info=data) return data diff --git a/nn_meter/builder/nn_meter_builder.py b/nn_meter/builder/nn_meter_builder.py index 303ec0d9..9babd72f 100644 --- a/nn_meter/builder/nn_meter_builder.py +++ b/nn_meter/builder/nn_meter_builder.py @@ -231,7 +231,7 @@ def build_predictor_for_kernel(kernel_type, backend, init_sample_num = 1000, fin def build_initial_predictor_by_data(kernel_type, backend = None, init_sample_num = 20, error_threshold = 0.1, mark = '', predict_label = "latency"): - return build_predictor_for_kernel(kernel_type, backend, init_sample_num=init_sample_num, iteration=1, error_threshold=error_threshold, predict_label=predict_label, mark=f'{mark}') + return build_predictor_for_kernel(kernel_type, backend, init_sample_num=init_sample_num, iteration=1, error_threshold=error_threshold, predict_label=predict_label, mark=mark) def build_adaptive_predictor_by_data(kernel_type, kernel_data, backend = None, finegrained_sample_num = 20, error_threshold = 0.1, mark = '', predict_label = "latency"): @@ -239,6 +239,7 @@ def build_adaptive_predictor_by_data(kernel_type, kernel_data, backend = None, f """ workspace_path = builder_config.get('WORKSPACE', 'predbuild') save_path = os.path.join(workspace_path, "results") + mark = 'finegrained' if mark == "" else f"finegrained_{mark}" from nn_meter.builder.kernel_predictor_builder import build_predictor_by_data, collect_kernel_data _, _, error_configs = build_predictor_by_data(kernel_type, kernel_data, backend = backend, error_threshold=error_threshold, save_path=None, predict_label=predict_label) @@ -246,10 +247,9 @@ def build_adaptive_predictor_by_data(kernel_type, kernel_data, backend = None, f sampling_mode='finegrained', configs=error_configs, mark=mark) # merge finegrained data with previous data and build new regression model - mark = mark if mark == "" else "_" + mark kernel_data = merge_info(new_info=new_kernel_data, prev_info=collect_kernel_data(kernel_data)) predictor, acc10, error_configs = build_predictor_by_data(kernel_type, kernel_data, backend, error_threshold=error_threshold, - mark=f'finegrained{mark}', save_path=save_path, predict_label=predict_label) + mark=mark, save_path=save_path, predict_label=predict_label) logging.keyinfo(f'{mark}: acc10 {acc10}, error_configs number: {len(error_configs)}') return predictor, kernel_data diff --git a/tests/unit_test/test_nn_modules.py b/tests/unit_test/test_nn_modules.py index 1062dde8..f350627e 100644 --- a/tests/unit_test/test_nn_modules.py +++ b/tests/unit_test/test_nn_modules.py @@ -7,7 +7,7 @@ if __name__ == '__main__': config = { - "HW": 1, + "HW": 24, "CIN": 144, "COUT": 32, "KERNEL_SIZE": 1, @@ -19,9 +19,12 @@ "CIN3": 12, "CIN4": 12 } + # test tersorflow kernels, tensorflow==2.7.0 or 2.6.0 is needed from nn_meter.builder.nn_modules.tf_networks import blocks for kernel in kernels: getattr(blocks, kernel)(config).test_block() + + # test torch kernels, torch==1.10.0 or 1.9.0 is needed from nn_meter.builder.nn_modules.torch_networks import blocks for kernel in kernels: getattr(blocks, kernel)(config).test_block() From cd9bcb3ce254d6defc4cdd2604f1679a931edec7 Mon Sep 17 00:00:00 2001 From: jiahangxu Date: Mon, 27 Jun 2022 05:17:20 -0400 Subject: [PATCH 3/3] complete v2.0 test --- docs/builder/build_kernel_latency_predictor.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/builder/build_kernel_latency_predictor.md b/docs/builder/build_kernel_latency_predictor.md index 2f0dfc4e..b9ef140d 100644 --- a/docs/builder/build_kernel_latency_predictor.md +++ b/docs/builder/build_kernel_latency_predictor.md @@ -244,7 +244,7 @@ predictor, data = build_predictor_for_kernel( ) ``` -In the experiment of nn-Meter, we set `init_sample_num` as 1000, `finegrained_sample_num` as 10, `iteration` as 5, and `error_threshold` as 0.1. +In the experiment of nn-Meter, we set default `init_sample_num` as 1000, `finegrained_sample_num` as 10, `iteration` as 5, and `error_threshold` as 0.1. nn-Meter also provided a end-to-end method for users to build a series of general latency predictors, named `nn_meter.builder.build_latency_predictor`. This method will build predictors for all kernels in `/configs/predictorbuild_config.yaml` according to their corresponding parameters. The parameters includes `INIT_SAMPLE_NUM`, `FINEGRAINED_SAMPLE_NUM`, `ITERATION`, and `ERROR_THRESHOLD`. Here is an example: diff --git a/setup.py b/setup.py index 296ae736..c0696fbe 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='nn-meter', - version='2.0a1', + version='2.0', description='nn-Meter is a novel and efficient system to accurately predict the inference latency of DNN models on diverse edge devices.', long_description = open('README.md', encoding='utf-8').read(), long_description_content_type = 'text/markdown',