diff --git a/docs/conf.py b/docs/conf.py index 7d4281f..5ade1da 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,7 +32,7 @@ # The short X.Y version version = '' # The full version, including alpha/beta/rc tags -release = '3.3.1' +release = '3.3.2' # -- General configuration --------------------------------------------------- diff --git a/ecnet/__init__.py b/ecnet/__init__.py index 40f5209..b2eedec 100644 --- a/ecnet/__init__.py +++ b/ecnet/__init__.py @@ -1,2 +1,2 @@ from ecnet.server import Server -__version__ = '3.3.1' +__version__ = '3.3.2' diff --git a/ecnet/models/mlp.py b/ecnet/models/mlp.py index 7ba06ce..e2da592 100644 --- a/ecnet/models/mlp.py +++ b/ecnet/models/mlp.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/models/mlp.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains the "MultilayerPerceptron" (feed-forward neural network) class diff --git a/ecnet/server.py b/ecnet/server.py index ed96467..80a2ae6 100644 --- a/ecnet/server.py +++ b/ecnet/server.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/server.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains the "Server" class, which handles ECNet project creation, neural @@ -151,7 +151,7 @@ def limit_inputs(self, limit_num: int, num_estimators: int = None, def tune_hyperparameters(self, num_employers: int, num_iterations: int, shuffle: bool = None, split: list = None, validate: bool = True, eval_set: str = None, - eval_fn: str = 'rmse', epochs: int = 300): + eval_fn: str = 'rmse', epochs: int = 500): '''Tunes neural network learning hyperparameters using an artificial bee colony algorithm; tuned hyperparameters are saved to Server's model configuration file @@ -167,7 +167,7 @@ def tune_hyperparameters(self, num_employers: int, num_iterations: int, `train`, `test`, None (all sets) eval_fn (str): error function used to evaluate bee fitness; `rmse`, `mean_abs_error`, `med_abs_error` - epochs (int): number of training epochs per bee ANN (def: 300) + epochs (int): number of training epochs per bee ANN (def: 500) ''' self._vars = tune_hyperparameters( diff --git a/ecnet/tasks/limit_inputs.py b/ecnet/tasks/limit_inputs.py index 376b714..85c38a4 100644 --- a/ecnet/tasks/limit_inputs.py +++ b/ecnet/tasks/limit_inputs.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/tasks/limit_inputs.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains functions for selecting influential input parameters diff --git a/ecnet/tasks/training.py b/ecnet/tasks/training.py index b762048..ecf39f9 100644 --- a/ecnet/tasks/training.py +++ b/ecnet/tasks/training.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/tasks/training.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains function for project training (multiprocessed training) diff --git a/ecnet/tasks/tuning.py b/ecnet/tasks/tuning.py index d7337ff..738fb00 100644 --- a/ecnet/tasks/tuning.py +++ b/ecnet/tasks/tuning.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/tasks/tuning.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains functions/fitness functions for tuning hyperparameters @@ -13,7 +13,7 @@ from os import name # 3rd party imports -from ecabc.abc import ABC +from ecabc import ABC # ECNet imports from ecnet.utils.data_utils import DataFrame @@ -25,7 +25,7 @@ def tune_hyperparameters(df: DataFrame, vars: dict, num_employers: int, num_iterations: int, num_processes: int = 1, shuffle: str = None, split: list = None, validate: bool = True, eval_set: str = None, - eval_fn: str = 'rmse', epochs: int = 300) -> dict: + eval_fn: str = 'rmse', epochs: int = 500) -> dict: '''Tunes neural network learning/architecture hyperparameters Args: @@ -41,7 +41,7 @@ def tune_hyperparameters(df: DataFrame, vars: dict, num_employers: int, `train`, `test`, None (all sets) eval_fn (str): error function used to evaluate bee performance; `rmse`, `mean_abs_error`, `med_abs_error` - epochs (int): number of training epochs per bee ANN (def: 300) + epochs (int): number of training epochs per bee ANN (def: 500) Returns: dict: tuned hyperparameters @@ -72,57 +72,48 @@ def tune_hyperparameters(df: DataFrame, vars: dict, num_employers: int, 'epochs': epochs } - value_ranges = [ - ('float', (1e-9, 1e-4)), # Learning rate decay - ('float', (1e-5, 0.1)), # Learning rate - ('int', (1, len(df.learn_set))), # Batch size - ('int', (64, 1024)) # Patience + to_tune = [ + (1e-9, 1e-4, 'decay'), + (1e-5, 0.1, 'learning_rate'), + (1, len(df.learn_set), 'batch_size'), + (64, 1024, 'patience') ] + for hl in range(len(vars['hidden_layers'])): + to_tune.append((1, 2 * len(df._input_names), 'hl{}'.format(hl))) - for _ in range(len(vars['hidden_layers'])): - value_ranges.append(('int', (1, len(df._input_names)))) - - abc = ABC( - tune_fitness_function, - num_employers=num_employers, - value_ranges=value_ranges, - args=fit_fn_args, - processes=num_processes - ) - - abc._logger.stream_level = logger.stream_level - if logger.file_level != 'disable': - abc._logger.log_dir = logger.log_dir - abc._logger.file_level = logger.file_level - abc._logger.default_call_loc('TUNE') - abc.create_employers() + abc = ABC(num_employers, tune_fitness_function, fit_fn_args, num_processes) + for param in to_tune: + abc.add_param(param[0], param[1], name=param[2]) + abc.initialize() + + best_ret_val = abc.best_ret_val + best_params = abc.best_params for i in range(num_iterations): logger.log('info', 'Iteration {}'.format(i + 1), call_loc='TUNE') - abc.run_iteration() + abc.search() + new_best_ret = abc.best_ret_val + new_best_params = abc.best_params logger.log('info', 'Best Performer: {}, {}'.format( - abc.best_performer[2], { - 'decay': abc.best_performer[1][0], - 'learning_rate': abc.best_performer[1][1], - 'batch_size': abc.best_performer[1][2], - 'patience': abc.best_performer[1][3], - 'hidden_layers': abc.best_performer[1][4:] - } + new_best_ret, new_best_ret ), call_loc='TUNE') - params = abc.best_performer[1] - vars['decay'] = params[0] - vars['learning_rate'] = params[1] - vars['batch_size'] = params[2] - vars['patience'] = params[3] + if new_best_ret < best_ret_val: + best_ret_val = new_best_ret + best_params = new_best_params + + vars['decay'] = best_params['decay'] + vars['learning_rate'] = best_params['learning_rate'] + vars['batch_size'] = best_params['batch_size'] + vars['patience'] = best_params['patience'] for l_idx in range(len(vars['hidden_layers'])): - vars['hidden_layers'][l_idx][0] = params[4 + l_idx] + vars['hidden_layers'][l_idx][0] = best_params['hl{}'.format(l_idx)] return vars -def tune_fitness_function(params: dict, **kwargs): +def tune_fitness_function(params: list, **kwargs) -> float: '''Fitness function used by ABC Args: - params (dict): bee hyperparams + params (list): bee hyperparams kwargs (dict): additional arguments Returns: diff --git a/ecnet/tools/database.py b/ecnet/tools/database.py index 6e7571d..3c147f1 100644 --- a/ecnet/tools/database.py +++ b/ecnet/tools/database.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/tools/database.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains functions for creating ECNet-formatted databases diff --git a/ecnet/tools/plotting.py b/ecnet/tools/plotting.py index 18de0c8..1fb6c44 100644 --- a/ecnet/tools/plotting.py +++ b/ecnet/tools/plotting.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/tools/plotting.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains functions/classes for creating various plots diff --git a/ecnet/tools/project.py b/ecnet/tools/project.py index 0b07b6c..0ab2171 100644 --- a/ecnet/tools/project.py +++ b/ecnet/tools/project.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/tools/project.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains functions for predicting data using pre-existing .prj files diff --git a/ecnet/utils/data_utils.py b/ecnet/utils/data_utils.py index 8b50c9b..735f511 100644 --- a/ecnet/utils/data_utils.py +++ b/ecnet/utils/data_utils.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/utils/data_utils.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains functions/classes for loading data, saving data, saving results diff --git a/ecnet/utils/error_utils.py b/ecnet/utils/error_utils.py index b699519..269d3f8 100644 --- a/ecnet/utils/error_utils.py +++ b/ecnet/utils/error_utils.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/utils/error_utils.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains functions for error calculations diff --git a/ecnet/utils/logging.py b/ecnet/utils/logging.py index 0ec25c0..d5fe6e6 100644 --- a/ecnet/utils/logging.py +++ b/ecnet/utils/logging.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/utils/logging.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains logger used by ECNet diff --git a/ecnet/utils/server_utils.py b/ecnet/utils/server_utils.py index c996134..08d0f1c 100644 --- a/ecnet/utils/server_utils.py +++ b/ecnet/utils/server_utils.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/utils/server_utils.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Contains functions used by ecnet.Server diff --git a/ecnet/workflows/ecrl_workflow.py b/ecnet/workflows/ecrl_workflow.py index f1740ae..93cf393 100644 --- a/ecnet/workflows/ecrl_workflow.py +++ b/ecnet/workflows/ecrl_workflow.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/workflows/ecrl_workflow.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # General workflow used by the UMass Lowell Energy and Combustion Research diff --git a/ecnet/workflows/workflow_utils.py b/ecnet/workflows/workflow_utils.py index 5629649..7fc3165 100644 --- a/ecnet/workflows/workflow_utils.py +++ b/ecnet/workflows/workflow_utils.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ecnet/workflows/workflow_utils.py -# v.3.3.1 +# v.3.3.2 # Developed in 2020 by Travis Kessler # # Functions used by the ECRL workflow diff --git a/setup.py b/setup.py index da7eaf5..13e3d05 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ install_requires = [ 'alvadescpy==0.1.0', 'colorlogging==0.3.5', - 'ecabc==2.2.3', + 'ecabc==3.0.0', 'matplotlib==3.1.2', 'numpy==1.16.4', 'padelpy==0.1.6', @@ -19,7 +19,7 @@ setup( name='ecnet', - version='3.3.1', + version='3.3.2', description='UMass Lowell Energy and Combustion Research Laboratory Neural' ' Network Software', url='https://github.com/ecrl/ecnet', diff --git a/tests/tasks/test_tune_hyperparameters.py b/tests/tasks/test_tune_hyperparameters.py index 81d97cb..cf46e22 100644 --- a/tests/tasks/test_tune_hyperparameters.py +++ b/tests/tasks/test_tune_hyperparameters.py @@ -17,7 +17,7 @@ def test_tune_hyperparameters(self): df = DataFrame(DB_LOC) df.create_sets(random=True) config = default_config() - new_hp = tune_hyperparameters(df, config, 2, 1) + new_hp = tune_hyperparameters(df, config, 2, 1, epochs=100) self.assertGreaterEqual(new_hp['beta_1'], 0) self.assertLessEqual(new_hp['beta_1'], 1) self.assertGreaterEqual(new_hp['beta_2'], 0) @@ -31,9 +31,33 @@ def test_tune_hyperparameters(self): self.assertGreaterEqual(new_hp['batch_size'], 1) self.assertLessEqual(new_hp['batch_size'], len(df.learn_set)) self.assertGreaterEqual(new_hp['hidden_layers'][0][0], 1) - self.assertLessEqual(new_hp['hidden_layers'][0][0], 100) + self.assertLessEqual(new_hp['hidden_layers'][0][0], 600) self.assertGreaterEqual(new_hp['hidden_layers'][1][0], 1) - self.assertLessEqual(new_hp['hidden_layers'][1][0], 100) + self.assertLessEqual(new_hp['hidden_layers'][1][0], 600) + + def test_th_multiprocess(self): + + print('\nUNIT TEST: tune_hyperparameters multiprocessed') + df = DataFrame(DB_LOC) + df.create_sets(random=True) + config = default_config() + new_hp = tune_hyperparameters(df, config, 2, 1, 2, epochs=100) + self.assertGreaterEqual(new_hp['beta_1'], 0) + self.assertLessEqual(new_hp['beta_1'], 1) + self.assertGreaterEqual(new_hp['beta_2'], 0) + self.assertLessEqual(new_hp['beta_2'], 1) + self.assertGreaterEqual(new_hp['decay'], 0) + self.assertLessEqual(new_hp['decay'], 1) + self.assertGreaterEqual(new_hp['epsilon'], 0) + self.assertLessEqual(new_hp['epsilon'], 1) + self.assertGreaterEqual(new_hp['learning_rate'], 0) + self.assertLessEqual(new_hp['learning_rate'], 1) + self.assertGreaterEqual(new_hp['batch_size'], 1) + self.assertLessEqual(new_hp['batch_size'], len(df.learn_set)) + self.assertGreaterEqual(new_hp['hidden_layers'][0][0], 1) + self.assertLessEqual(new_hp['hidden_layers'][0][0], 600) + self.assertGreaterEqual(new_hp['hidden_layers'][1][0], 1) + self.assertLessEqual(new_hp['hidden_layers'][1][0], 600) if __name__ == '__main__':