Refactor: split genoptimizer into multiple files.

mloptimizer/evaluation/__init__.py

@@ -0,0 +1,2 @@
+from .model_evaluation import kfold_stratified_score, temporal_kfold_score, \
+    train_score, train_test_score, kfold_score

mloptimizer/genoptimizer/__init__.py

@@ -0,0 +1,7 @@
+from .hyperparam import Hyperparam
+from .base import BaseOptimizer
+from .trees import TreeOptimizer, ForestOptimizer, ExtraTreesOptimizer, GradientBoostingOptimizer
+from .xgb import XGBClassifierOptimizer, CustomXGBClassifierOptimizer
+from .svc import SVCOptimizer
+from .keras import KerasClassifierOptimizer
+from .catboost import CatBoostClassifierOptimizer

mloptimizer/genoptimizer/catboost.py

@@ -0,0 +1,29 @@
+from abc import ABC
+from catboost import CatBoostClassifier
+
+from mloptimizer.genoptimizer import Hyperparam, BaseOptimizer
+
+
+class CatBoostClassifierOptimizer(BaseOptimizer, ABC):
+    """
+    Class for the optimization of a gradient boosting classifier from catboost.CatBoostClassifier.
+    It inherits from BaseOptimizer.
+    """
+
+    @staticmethod
+    def get_default_hyperparams():
+        default_hyperparams = {
+            'eta': Hyperparam("eta", 1, 10, float, 10),
+            'max_depth': Hyperparam("max_depth", 3, 16, int),  # Max is 16
+            'n_estimators': Hyperparam("n_estimators", 100, 500, int),
+            'subsample': Hyperparam("subsample", 700, 1000, float, 1000),
+        }
+        return default_hyperparams
+
+    def get_clf(self, individual):
+        individual_dict = self.individual2dict(individual)
+        clf = CatBoostClassifier(
+            **individual_dict, auto_class_weights="Balanced",
+            bootstrap_type='Bernoulli'
+        )
+        return clf

mloptimizer/genoptimizer/hyperparam.py

@@ -0,0 +1,125 @@
+class Hyperparam(object):
+    """
+    Class to define a hyperparam to optimize. It defines the name, min value, max value and type.
+    This is used to control the precision of the hyperparam and avoid multiple evaluations
+    with close values of the hyperparam due to decimal positions.
+
+
+    Attributes
+    ----------
+    name : str
+        Name of the hyperparam. It will be used as key in a dictionary
+    min_value : int
+        Minimum value of the hyperparam
+    max_value : int
+        Maximum value of the hyperparam
+    type : type
+        Type of the hyperparam (int, float, 'nexp', 'x10')
+    denominator : int, optional (default=100)
+        Optional param in case the type=float
+    values_str : list, optional (default=[])
+        List of string with possible values (TODO)
+    """
+
+    def __init__(self, name: str, min_value: int, max_value: int, hyperparam_type,
+                 denominator: int = 100, values_str: list = None):
+        """
+        Creates object Hyperparam.
+
+        Parameters
+        ----------
+        name : str
+            Name of the hyperparam. It will be used as key in a dictionary
+        min_value : int
+            Minimum value of the hyperparam
+        max_value : int
+            Maximum value of the hyperparam
+        type : type
+            Type of the hyperparam (int, float, 'nexp', 'x10')
+        denominator : int, optional (default=100)
+            Optional param in case the type=float
+        values_str : list, optional (default=[])
+            List of string with possible values (TODO)
+        """
+        if values_str is None:
+            values_str = []
+        self.name = name
+        self.min_value = min_value
+        self.max_value = max_value
+        self.type = hyperparam_type
+        self.denominator = denominator
+        self.values_str = values_str
+
+    def correct(self, value: int):
+        """
+        Returns the real value of the hyperparam in case some mutation could surpass the limits.
+            1) Verifies the input is int
+            2) Enforce min and max value
+            3) Apply the type of value
+
+        Parameters
+        ----------
+        value : int
+            Value to correct
+
+        Returns
+        -------
+        ret : int, float
+            Corrected value
+        """
+        # Input value must be int
+        value = int(value)
+        ret = None
+        # Verify the value is in range
+        if value > self.max_value:
+            value = self.max_value
+        elif value < self.min_value:
+            value = self.min_value
+        # Apply the type of value
+        if self.type == int:
+            ret = value
+        elif self.type == float:
+            ret = float(value) / self.denominator
+            # ret = round(value, self.decimals)
+        elif self.type == "nexp":
+            ret = 10 ** (-value)
+        elif self.type == "x10":
+            ret = value * 10
+        return ret
+
+    def __eq__(self, other_hyperparam):
+        """Overrides the default implementation"""
+        equals = (self.name == other_hyperparam.name and self.min_value == other_hyperparam.min_value and
+                  self.type == other_hyperparam.type and self.denominator == other_hyperparam.denominator and
+                  self.max_value == other_hyperparam.max_value)
+        return equals
+
+    def __str__(self):
+        """Overrides the default implementation"""
+        if self.type is str:
+            type_str = "'{}'".format(self.type)
+        else:
+            type_str = self.type.__name__
+
+        if self.type == float:
+            hyperparam_str = "Hyperparam('{}', {}, {}, {}, {})".format(
+                self.name,
+                self.min_value,
+                self.max_value,
+                type_str,
+                self.denominator
+            )
+        else:
+            hyperparam_str = "Hyperparam('{}', {}, {}, {})".format(
+                self.name,
+                self.min_value,
+                self.max_value,
+                type_str
+            )
+
+        return hyperparam_str
+
+    def __repr__(self):
+        """Overrides the default implementation"""
+        return self.__str__()
+

mloptimizer/genoptimizer/keras.py

@@ -0,0 +1,32 @@
+from abc import ABC
+from keras.wrappers.scikit_learn import KerasClassifier
+
+from mloptimizer.alg_wrapper import generate_model
+from mloptimizer.genoptimizer import Hyperparam, BaseOptimizer
+
+
+class KerasClassifierOptimizer(BaseOptimizer, ABC):
+    """
+    Class for the optimization of a gradient boosting classifier from keras.wrappers.scikit_learn.KerasClassifier.
+    It inherits from BaseOptimizer.
+    """
+
+    @staticmethod
+    def get_default_hyperparams():
+        default_hyperparams = {
+            'epochs': Hyperparam("epochs", 1, 10, "x10"),
+            'batch_size': Hyperparam("batch_size", 1, 5, "x10"),
+            'learning_rate': Hyperparam("learning_rate", 1, 20, float, 1000),
+            'layer_1': Hyperparam("layer_1", 10, 50, "x10"),
+            'layer_2': Hyperparam("layer_2", 5, 20, "x10"),
+            'dropout_rate_1': Hyperparam("dropout_rate_1", 0, 5, float, 10),
+            'dropout_rate_2': Hyperparam("dropout_rate_2", 0, 5, float, 10),
+        }
+        return default_hyperparams
+
+    def get_clf(self, individual):
+        individual_dict = self.individual2dict(individual)
+        print(individual_dict)
+        clf = KerasClassifier(build_fn=generate_model,
+                              **individual_dict)
+        return clf

mloptimizer/genoptimizer/svc.py

@@ -0,0 +1,38 @@
+from abc import ABC
+from sklearn.svm import SVC
+
+from mloptimizer.genoptimizer import Hyperparam, BaseOptimizer
+
+
+class SVCOptimizer(BaseOptimizer, ABC):
+    """
+    Class for the optimization of a support vector machine classifier from sklearn.svm.SVC.
+    It inherits from BaseOptimizer.
+    """
+
+    @staticmethod
+    def get_default_hyperparams():
+        default_hyperparams = {
+            'C': Hyperparam("C", 1, 10000, float, 10),
+            'degree': Hyperparam("degree", 0, 6, int),
+            'gamma': Hyperparam("gamma", 10, 100000000, float, 100)
+        }
+        return default_hyperparams
+
+    def get_clf(self, individual):
+        individual_dict = self.individual2dict(individual)
+        clf = SVC(C=individual_dict['C'],
+                  cache_size=8000000,
+                  class_weight="balanced",
+                  coef0=0.0,
+                  decision_function_shape='ovr',
+                  degree=individual_dict['degree'], gamma=individual_dict['gamma'],
+                  kernel='rbf',
+                  max_iter=100000,
+                  probability=False,
+                  random_state=None,
+                  shrinking=True,
+                  tol=0.001,
+                  verbose=False
+                  )
+        return clf