From 0c0f38244fc73a7493fa707b2891a2c100957c26 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Tue, 5 Jul 2022 18:16:24 +0100 Subject: [PATCH 01/50] SSPT stub --- river_extra/model_selection/__init__.py | 0 river_extra/model_selection/sspt.py | 123 ++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 river_extra/model_selection/__init__.py create mode 100644 river_extra/model_selection/sspt.py diff --git a/river_extra/model_selection/__init__.py b/river_extra/model_selection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py new file mode 100644 index 0000000..059a924 --- /dev/null +++ b/river_extra/model_selection/sspt.py @@ -0,0 +1,123 @@ +import abc +import collections +import typing + +from river import base, drift, metrics + +ModelWrapper = collections.namedtuple("ModelWrapper", "model metric") + + +# TODO: change class inheritance +class SSPT(base.Estimator): + def __init__( + self, + model, + metric: metrics.base.Metric, + params: typing.Dict[str, typing.Tuple], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + ): + self.model = model + self.metric = metric + self.params = params + + self.grace_period = grace_period + self.drift_detector = drift_detector + + self._best_model = None + + # [best, good, worst] + self._simplex = self._create_simplex(model) + + # Models expanded from the simplex + self._expanded: typing.Optional[typing.List] = None + + self._n = 0 + self._converged = False + + def _create_simplex(self, model) -> typing.List: + # TODO use namedtuple ModelWrapper to wrap model and performance metric + pass + + def _expand_simplex(self) -> typing.List: + # TODO Here happens the model expansion + pass + + @property + def _models_converged(self) -> bool: + # TODO check convergence criteria + return False + + @abc.abstractmethod + def _drift_input(self, y_true, y_pred) -> typing.Union[int, float]: + pass + + def _learn_converged(self, x, y): + y_pred = self._best_model.predict_one(x) + + input = self._drift_input(y, y_pred) + self.drift_detector.update(input) + + # We need to start the optimization process from scratch + if self.drift_detector.drift_detected: + self._converged = False + self._simplex = self._create_simplex(self._best_model) + + # There is no proven best model right now + self._best_model = None + return + + self._best_model.learn_one(x, y) + + def _learn_not_converged(self, x, y): + for wrap in self._simplex: + y_pred = wrap.model.predict_one(x) + wrap.metric.update(y, y_pred) + wrap.model.learn_one(x, y) + + if not self._expanded: + self._expanded = self._expand_simplex() + + for wrap in self._expanded: + y_pred = wrap.model.predict_one(x) + wrap.metric.update(y, y_pred) + wrap.model.learn_one(x, y) + + if self._n == self.grace_period: + self._n = 0 + + # Take the best expanded model and replace the worst contender in the simplex + if self.metric.bigger_is_better: + self._expanded.sort(key=lambda w: w.metric.get(), reverse=True) + self._simplex[-1] = self._expanded[0] + self._simplex.sort(key=lambda w: w.metric.get(), reverse=True) + else: + self._expanded.sort(key=lambda w: w.metric.get()) + self._simplex[-1] = self._expanded[0] + self._simplex.sort(key=lambda w: w.metric.get()) + + # Discard expanded models + self._expanded = None + + if self._models_converged: + self._converged = True + self._best_model = self._simplex[0].model + + def learn_one(self, x, y): + self._n += 1 + + if self.converged: + self._learn_converged(x, y) + else: + self._learn_not_converged(x, y) + + @property + def best_model(self): + if not self._converged: + return self._simplex[0].model + + return self._best_model + + @property + def converged(self): + return self._converged From 822beda28b5e47bf9d7ba677ac88329aaf2b6852 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Wed, 6 Jul 2022 17:02:36 +0100 Subject: [PATCH 02/50] Finish stub --- river_extra/model_selection/sspt.py | 197 ++++++++++++++++++++++++---- 1 file changed, 174 insertions(+), 23 deletions(-) diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 059a924..32c7207 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -1,51 +1,206 @@ import abc import collections +import math +import random import typing -from river import base, drift, metrics +from river import base, drift, metrics, utils ModelWrapper = collections.namedtuple("ModelWrapper", "model metric") # TODO: change class inheritance class SSPT(base.Estimator): + _START_RANDOM = "random" + _START_WARM = "warm" + def __init__( self, model, metric: metrics.base.Metric, - params: typing.Dict[str, typing.Tuple], + params_range: typing.Dict[str, typing.Tuple], grace_period: int = 500, drift_detector: base.DriftDetector = drift.ADWIN(), + start: str = "warm", + convergence_sphere: float = 0.01, + seed: int = None, ): self.model = model self.metric = metric - self.params = params + self.params_range = params_range self.grace_period = grace_period self.drift_detector = drift_detector - self._best_model = None + if start not in {self._START_RANDOM, self._START_WARM}: + raise ValueError( + f"'start' must be either '{self._START_RANDOM}' or '{self._START_WARM}'." + ) + self.start = start + self.convergence_sphere = convergence_sphere + + self.seed = seed - # [best, good, worst] + self._best_model = None self._simplex = self._create_simplex(model) # Models expanded from the simplex - self._expanded: typing.Optional[typing.List] = None + self._expanded: typing.Optional[typing.Dict] = None self._n = 0 self._converged = False + self._rng = random.Random(self.seed) + + def _random_config(self): + config = {} + + for p_name, (p_type, p_range) in self.params_range.items(): + if p_type == int: + config[p_name] = self._rng.randint(p_range[0], p_range[1]) + elif p_type == float: + config[p_name] = self._rng.uniform(p_range[0], p_range[1]) + + return config def _create_simplex(self, model) -> typing.List: - # TODO use namedtuple ModelWrapper to wrap model and performance metric - pass + # The simplex is divided in: + # * 0: the best model + # * 1: the 'good' model + # * 2: the worst model + simplex = [None] * 3 - def _expand_simplex(self) -> typing.List: - # TODO Here happens the model expansion - pass + simplex[0] = ModelWrapper( + self.model.clone(self._random_config()), self.metric.clone() + ) + simplex[2] = ModelWrapper( + self.model.clone(self._random_config()), self.metric.clone() + ) + + g_params = model._get_params() + if self.start == self._START_RANDOM: + # The intermediate 'good' model is defined randomly + g_params = self._random_config() + + simplex[1] = ModelWrapper(self.model.clone(g_params), self.metric.clone()) + + return simplex + + def _sort_simplex(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._simplex.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._simplex.sort(key=lambda mw: mw.metric.get()) + + def _nelder_mead_expansion(self) -> typing.Dict: + """Create expanded models given the simplex models.""" + + def apply_operator(m1, m2, func): + new_config = {} + m1_params = m1.model._get_params() + m2_params = m2.model._get_params() + + for p_name, (p_type, p_range) in self.params_range.items(): + new_val = func(m1_params[p_name], m2_params[p_name]) + + # Range sanity checks + if new_val < p_range[0]: + new_val = p_range[0] + if new_val > p_range[1]: + new_val = p_range[1] + + new_config[p_name] = round(new_val, 0) if p_type == int else new_val + + # Modify the current best contender with the new hyperparameter values + return ModelWrapper( + self._simplex[0].mutate(new_config), self.metric.clone() + ) + + expanded = {} + # Midpoint between 'best' and 'good' + expanded["midpoint"] = apply_operator( + self._simplex[0], self._simplex[1], lambda h1, h2: (h1 + h2) / 2 + ) + # Reflection of 'midpoint' towards 'worst' + expanded["reflection"] = apply_operator( + expanded["midpoint"], self._simplex[2], lambda h1, h2: 2 * h1 - h2 + ) + # Expand the 'reflection' point + expanded["expansion"] = apply_operator( + expanded["reflection"], expanded["midpoint"], lambda h1, h2: 2 * h1 - h2 + ) + # Shrink 'best' and 'worst' + expanded["shrink"] = apply_operator( + self._simplex[0], self._simplex[2], lambda h1, h2: (h1 + h2) / 2 + ) + # Contraction of 'midpoint' and 'worst' + expanded["contraction"] = apply_operator( + expanded["midpoint"], self._simplex[2], lambda h1, h2: (h1 + h2) / 2 + ) + + return expanded + + def _nelder_mead_operators(self): + b = self._simplex[0].metric + g = self._simplex[1].metric + w = self._simplex[2].metric + r = self._expanded["reflection"].metric + + if r.is_better_than(g): + if b.is_better_than(r): + self._simplex[2] = self._expanded["reflection"] + else: + e = self._expanded["expansion"].metric + if e.is_better_than(b): + self._simplex[2] = self._expanded["expansion"] + else: + self._simplex[2] = self._expanded["reflection"] + else: + if r.is_better_than(w): + self._simplex[2] = self._expanded["reflection"] + else: + c = self._expanded["contraction"].metric + if c.is_better_than(w): + self._simplex[2] = self._expanded["contraction"] + else: + s = self._expanded["shrink"].metric + if s.is_better_than(w): + self._simplex[2] = self._expanded["shrink"] + m = self._expanded["midpoint"].metric + if m.is_better_than(g): + self._simplex[1] = self._expanded["midpoint"] @property def _models_converged(self) -> bool: - # TODO check convergence criteria + # 1. Simplex in sphere + + params_b = self._simplex[0].model._get_params() + params_g = self._simplex[1].model._get_params() + params_w = self._simplex[2].model._get_params() + + # Normalize params to ensure the contribute equally to the stopping criterion + for p_name, (_, p_range) in self.params_range.items(): + scale = p_range[1] - p_range[0] + params_b[p_name] = (params_b[p_name] - p_range[0]) / scale + params_g[p_name] = (params_g[p_name] - p_range[0]) / scale + params_w[p_name] = (params_w[p_name] - p_range[0]) / scale + + max_dist = max( + [ + utils.math.minkowski_distance(params_b, params_g, p=2), + utils.math.minkowski_distance(params_b, params_w, p=2), + utils.math.minkowski_distance(params_g, params_w, p=2), + ] + ) + + ndim = len(params_b) + r_sphere = max_dist * math.sqrt((ndim / (2 * (ndim + 1)))) + + if r_sphere < self.convergence_sphere: + return True + + # TODO? 2. Simplex did not change + return False @abc.abstractmethod @@ -76,9 +231,9 @@ def _learn_not_converged(self, x, y): wrap.model.learn_one(x, y) if not self._expanded: - self._expanded = self._expand_simplex() + self._expanded = self._nelder_mead_expansion() - for wrap in self._expanded: + for wrap in self._expanded.values(): y_pred = wrap.model.predict_one(x) wrap.metric.update(y, y_pred) wrap.model.learn_one(x, y) @@ -86,15 +241,9 @@ def _learn_not_converged(self, x, y): if self._n == self.grace_period: self._n = 0 - # Take the best expanded model and replace the worst contender in the simplex - if self.metric.bigger_is_better: - self._expanded.sort(key=lambda w: w.metric.get(), reverse=True) - self._simplex[-1] = self._expanded[0] - self._simplex.sort(key=lambda w: w.metric.get(), reverse=True) - else: - self._expanded.sort(key=lambda w: w.metric.get()) - self._simplex[-1] = self._expanded[0] - self._simplex.sort(key=lambda w: w.metric.get()) + self._sort_simplex() + # Update the simplex models using Nelder-Mead heuristics + self._nelder_mead_operators() # Discard expanded models self._expanded = None @@ -114,6 +263,8 @@ def learn_one(self, x, y): @property def best_model(self): if not self._converged: + # Lazy selection of the best model + self._sort_simplex() return self._simplex[0].model return self._best_model From d78c66b1e2ad60e4149f05baf7e83ce456855a08 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Wed, 6 Jul 2022 17:59:12 +0100 Subject: [PATCH 03/50] checkpoint --- river_extra/model_selection/__init__.py | 3 + river_extra/model_selection/sspt.py | 134 ++++++++++++++++++++---- 2 files changed, 114 insertions(+), 23 deletions(-) diff --git a/river_extra/model_selection/__init__.py b/river_extra/model_selection/__init__.py index e69de29..9c096e3 100644 --- a/river_extra/model_selection/__init__.py +++ b/river_extra/model_selection/__init__.py @@ -0,0 +1,3 @@ +from .sspt import SSPTRegressor + +__all__ = ["SSPTRegressor"] diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 32c7207..39fb716 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -1,5 +1,6 @@ import abc import collections +import copy import math import random import typing @@ -11,20 +12,23 @@ # TODO: change class inheritance class SSPT(base.Estimator): + """Single-pass Self Parameter Tuning""" + _START_RANDOM = "random" _START_WARM = "warm" def __init__( self, model, - metric: metrics.base.Metric, + metric, params_range: typing.Dict[str, typing.Tuple], - grace_period: int = 500, - drift_detector: base.DriftDetector = drift.ADWIN(), - start: str = "warm", - convergence_sphere: float = 0.01, - seed: int = None, + grace_period: int, + drift_detector: base.DriftDetector, + start: str, + convergence_sphere: float, + seed: int, ): + super().__init__() self.model = model self.metric = metric self.params_range = params_range @@ -41,16 +45,16 @@ def __init__( self.seed = seed + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + self._best_model = None self._simplex = self._create_simplex(model) # Models expanded from the simplex self._expanded: typing.Optional[typing.Dict] = None - self._n = 0 - self._converged = False - self._rng = random.Random(self.seed) - def _random_config(self): config = {} @@ -112,15 +116,19 @@ def apply_operator(m1, m2, func): new_config[p_name] = round(new_val, 0) if p_type == int else new_val # Modify the current best contender with the new hyperparameter values - return ModelWrapper( - self._simplex[0].mutate(new_config), self.metric.clone() + new = ModelWrapper( + copy.deepcopy(self._simplex[0].model), self.metric.clone() ) + new.model.mutate(new_config) + + return new expanded = {} # Midpoint between 'best' and 'good' expanded["midpoint"] = apply_operator( self._simplex[0], self._simplex[1], lambda h1, h2: (h1 + h2) / 2 ) + # Reflection of 'midpoint' towards 'worst' expanded["reflection"] = apply_operator( expanded["midpoint"], self._simplex[2], lambda h1, h2: 2 * h1 - h2 @@ -169,31 +177,36 @@ def _nelder_mead_operators(self): m = self._expanded["midpoint"].metric if m.is_better_than(g): self._simplex[1] = self._expanded["midpoint"] + + self._sort_simplex() @property def _models_converged(self) -> bool: - # 1. Simplex in sphere - + # 1. Simplex in spher params_b = self._simplex[0].model._get_params() params_g = self._simplex[1].model._get_params() params_w = self._simplex[2].model._get_params() - # Normalize params to ensure the contribute equally to the stopping criterion + scaled_params_b = {} + scaled_params_g = {} + scaled_params_w = {} + + # Normalize params to ensure they contribute equally to the stopping criterion for p_name, (_, p_range) in self.params_range.items(): scale = p_range[1] - p_range[0] - params_b[p_name] = (params_b[p_name] - p_range[0]) / scale - params_g[p_name] = (params_g[p_name] - p_range[0]) / scale - params_w[p_name] = (params_w[p_name] - p_range[0]) / scale + scaled_params_b[p_name] = (params_b[p_name] - p_range[0]) / scale + scaled_params_g[p_name] = (params_g[p_name] - p_range[0]) / scale + scaled_params_w[p_name] = (params_w[p_name] - p_range[0]) / scale max_dist = max( [ - utils.math.minkowski_distance(params_b, params_g, p=2), - utils.math.minkowski_distance(params_b, params_w, p=2), - utils.math.minkowski_distance(params_g, params_w, p=2), + utils.math.minkowski_distance(scaled_params_b, scaled_params_g, p=2), + utils.math.minkowski_distance(scaled_params_b, scaled_params_w, p=2), + utils.math.minkowski_distance(scaled_params_g, scaled_params_w, p=2), ] ) - ndim = len(params_b) + ndim = len(self.params_range) r_sphere = max_dist * math.sqrt((ndim / (2 * (ndim + 1)))) if r_sphere < self.convergence_sphere: @@ -229,6 +242,9 @@ def _learn_not_converged(self, x, y): y_pred = wrap.model.predict_one(x) wrap.metric.update(y, y_pred) wrap.model.learn_one(x, y) + + # Keep the simplex ordered + self._sort_simplex() if not self._expanded: self._expanded = self._nelder_mead_expansion() @@ -241,7 +257,6 @@ def _learn_not_converged(self, x, y): if self._n == self.grace_period: self._n = 0 - self._sort_simplex() # Update the simplex models using Nelder-Mead heuristics self._nelder_mead_operators() @@ -272,3 +287,76 @@ def best_model(self): @property def converged(self): return self._converged + + +class SSPTRegressor(SSPT, base.Regressor): + """Single-pass Self Parameter Tuning Regressor. + + Parameters + ---------- + model + metric + params_range + grace_period + drift_detector + start + convergence_sphere + seed + + Examples + -------- + >>> from river import datasets + >>> from river import linear_model + >>> from river import metrics + >>> from river import preprocessing + >>> from river_extra import model_selection + + >>> dataset = datasets.synth.Friedman(seed=42).take(2000) + >>> reg = preprocessing.StandardScaler() | model_selection.SSPTRegressor( + model=linear_model.LinearRegressor(), + metric=metrics.RMSE(), + params_range={ + "l2": (float, (0.0, 0.5)) + } + ) + >>> metric = metrics.RMSE() + + >>> for x, y in dataset: + ... y_pred = reg.predict_one(x) + ... metric.update(y, y_pred) + ... reg.learn_one(x, y) + + >>> metric + + References + ---------- + [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021).Hyperparameter self-tuning + for data streams. Information Fusion, 76, 75-86. + """ + def __init__( + self, + model: base.Regressor, + metric: metrics.base.RegressionMetric, + params_range: typing.Dict[str, typing.Tuple], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + start: str = "warm", + convergence_sphere: float = 0.0001, + seed: int = None, + ): + super().__init__( + model, + metric, + params_range, + grace_period, + drift_detector, + start, + convergence_sphere, + seed, + ) + + def _drift_input(self, y_true, y_pred): + return abs(y_true - y_pred) + + def predict_one(self, x: dict): + return self.best_model.predict_one(x) From 091852430434c28dcff0cbe01f02adbe42027aae Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Wed, 6 Jul 2022 18:29:17 +0100 Subject: [PATCH 04/50] add SSPTClassifier --- river_extra/model_selection/__init__.py | 4 +- river_extra/model_selection/sspt.py | 52 +++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/river_extra/model_selection/__init__.py b/river_extra/model_selection/__init__.py index 9c096e3..1423b59 100644 --- a/river_extra/model_selection/__init__.py +++ b/river_extra/model_selection/__init__.py @@ -1,3 +1,3 @@ -from .sspt import SSPTRegressor +from .sspt import SSPTClassifier, SSPTRegressor -__all__ = ["SSPTRegressor"] +__all__ = ["SSPTClassifier", "SSPTRegressor"] diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 39fb716..c901b77 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -228,6 +228,7 @@ def _learn_converged(self, x, y): # We need to start the optimization process from scratch if self.drift_detector.drift_detected: + self._n = 0 self._converged = False self._simplex = self._create_simplex(self._best_model) @@ -274,6 +275,8 @@ def learn_one(self, x, y): self._learn_converged(x, y) else: self._learn_not_converged(x, y) + + return self @property def best_model(self): @@ -289,6 +292,55 @@ def converged(self): return self._converged + +class SSPTClassifier(SSPT, base.Classifier): + """Single-pass Self Parameter Tuning Regressor. + + Parameters + ---------- + model + metric + params_range + grace_period + drift_detector + start + convergence_sphere + seed + + References + ---------- + [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021).Hyperparameter self-tuning + for data streams. Information Fusion, 76, 75-86. + """ + def __init__( + self, + model: base.Classifier, + metric: metrics.base.ClassificationMetric, + params_range: typing.Dict[str, typing.Tuple], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + start: str = "warm", + convergence_sphere: float = 0.0001, + seed: int = None, + ): + super().__init__( + model, + metric, + params_range, + grace_period, + drift_detector, + start, + convergence_sphere, + seed, + ) + + def _drift_input(self, y_true, y_pred): + return 0 if y_true == y_pred else 1 + + def predict_proba_one(self, x: dict): + return self.best_model.predict_proba_one(x) + + class SSPTRegressor(SSPT, base.Regressor): """Single-pass Self Parameter Tuning Regressor. From 110274fe0613bf84cf2375defda8af34167aaf4f Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Thu, 7 Jul 2022 11:30:20 +0100 Subject: [PATCH 05/50] checkpoint --- river_extra/model_selection/sspt.py | 136 +++++++++++++++++++--------- 1 file changed, 94 insertions(+), 42 deletions(-) diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index c901b77..308df61 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -3,6 +3,7 @@ import copy import math import random +import types import typing from river import base, drift, metrics, utils @@ -10,7 +11,6 @@ ModelWrapper = collections.namedtuple("ModelWrapper", "model metric") -# TODO: change class inheritance class SSPT(base.Estimator): """Single-pass Self Parameter Tuning""" @@ -55,14 +55,31 @@ def __init__( # Models expanded from the simplex self._expanded: typing.Optional[typing.Dict] = None + # Meta-programming + self._bind_output_method() + + def _bind_output_method(self): + pass + + def _random_config(self): - config = {} + def gen_random(p_info): + if isinstance(p_info[-1], dict): + new_vals = {} + sub_class, sub_params = p_info + for p_name in sub_params: + new_vals[p_name] = gen_random(sub_params[p_name]) + return sub_class(**new_vals) + else: + p_type, p_range = p_info + if p_type == int: + return self._rng.randint(p_range[0], p_range[1]) + elif p_type == float: + return self._rng.uniform(p_range[0], p_range[1]) - for p_name, (p_type, p_range) in self.params_range.items(): - if p_type == int: - config[p_name] = self._rng.randint(p_range[0], p_range[1]) - elif p_type == float: - config[p_name] = self._rng.uniform(p_range[0], p_range[1]) + config = {} + for p_name, p_info in self.params_range.items(): + config[p_name] = gen_random(p_info) return config @@ -80,12 +97,14 @@ def _create_simplex(self, model) -> typing.List: self.model.clone(self._random_config()), self.metric.clone() ) - g_params = model._get_params() if self.start == self._START_RANDOM: # The intermediate 'good' model is defined randomly - g_params = self._random_config() - - simplex[1] = ModelWrapper(self.model.clone(g_params), self.metric.clone()) + simplex[1] = ModelWrapper( + self.model.clone(self._random_config()), self.metric.clone() + ) + elif self.start == self._START_WARM: + # The intermediate 'good' model is defined randomly + simplex[1] = ModelWrapper(copy.deepcopy(model), self.metric.clone()) return simplex @@ -99,13 +118,22 @@ def _sort_simplex(self): def _nelder_mead_expansion(self) -> typing.Dict: """Create expanded models given the simplex models.""" - def apply_operator(m1, m2, func): - new_config = {} - m1_params = m1.model._get_params() - m2_params = m2.model._get_params() - - for p_name, (p_type, p_range) in self.params_range.items(): - new_val = func(m1_params[p_name], m2_params[p_name]) + def apply_operator(param1, param2, func, p_info): + if isinstance(p_info[1], dict): + sub_class, sub_params1 = param1 + _, sub_params2 = param2 + sub_info = p_info[1] + new_params = {} + for sp_name in sub_params1: + sp_info = sub_info[sp_name] + # Recursive call to deal with nested hiperparameters + new_params[sp_name] = apply_operator( + sub_params1[sp_name], sub_params2[sp_name], func, sp_info + ) + return sub_class(**new_params) + else: + p_type, p_range = p_info + new_val = func(param1, param2) # Range sanity checks if new_val < p_range[0]: @@ -113,36 +141,47 @@ def apply_operator(m1, m2, func): if new_val > p_range[1]: new_val = p_range[1] - new_config[p_name] = round(new_val, 0) if p_type == int else new_val + new_val = round(new_val, 0) if p_type == int else new_val + + return new_val + + def gen_new(m1, m2, func): + new_config = {} + m1_params = m1.model._get_params() + m2_params = m2.model._get_params() + + for p_name, p_info in self.params_range.items(): + new_config[p_name] = apply_operator( + m1_params[p_name], m2_params[p_name], func, p_info + ) # Modify the current best contender with the new hyperparameter values new = ModelWrapper( copy.deepcopy(self._simplex[0].model), self.metric.clone() ) - new.model.mutate(new_config) return new expanded = {} # Midpoint between 'best' and 'good' - expanded["midpoint"] = apply_operator( + expanded["midpoint"] = gen_new( self._simplex[0], self._simplex[1], lambda h1, h2: (h1 + h2) / 2 ) # Reflection of 'midpoint' towards 'worst' - expanded["reflection"] = apply_operator( + expanded["reflection"] = gen_new( expanded["midpoint"], self._simplex[2], lambda h1, h2: 2 * h1 - h2 ) # Expand the 'reflection' point - expanded["expansion"] = apply_operator( + expanded["expansion"] = gen_new( expanded["reflection"], expanded["midpoint"], lambda h1, h2: 2 * h1 - h2 ) # Shrink 'best' and 'worst' - expanded["shrink"] = apply_operator( + expanded["shrink"] = gen_new( self._simplex[0], self._simplex[2], lambda h1, h2: (h1 + h2) / 2 ) # Contraction of 'midpoint' and 'worst' - expanded["contraction"] = apply_operator( + expanded["contraction"] = gen_new( expanded["midpoint"], self._simplex[2], lambda h1, h2: (h1 + h2) / 2 ) @@ -177,26 +216,38 @@ def _nelder_mead_operators(self): m = self._expanded["midpoint"].metric if m.is_better_than(g): self._simplex[1] = self._expanded["midpoint"] - + self._sort_simplex() @property def _models_converged(self) -> bool: - # 1. Simplex in spher - params_b = self._simplex[0].model._get_params() - params_g = self._simplex[1].model._get_params() - params_w = self._simplex[2].model._get_params() + # Normalize params to ensure they contribute equally to the stopping criterion + def normalize_flattened_hyperspace(scaled, orig, info, prefix=""): + for p_name, p_info in info.items(): + prefix_ = prefix + p_name + if isinstance(p_info[-1], dict): + sub_orig = orig[p_name][1] + sub_info = p_info[1] + prefix_ += "__" + normalize_flattened_hyperspace(scaled, sub_orig, sub_info, prefix_) + else: + _, p_range = p_info + interval = p_range[1] - p_range[0] + scaled[prefix_] = (orig[p_name] - p_range[0]) / interval + # 1. Simplex in sphere scaled_params_b = {} scaled_params_g = {} scaled_params_w = {} - - # Normalize params to ensure they contribute equally to the stopping criterion - for p_name, (_, p_range) in self.params_range.items(): - scale = p_range[1] - p_range[0] - scaled_params_b[p_name] = (params_b[p_name] - p_range[0]) / scale - scaled_params_g[p_name] = (params_g[p_name] - p_range[0]) / scale - scaled_params_w[p_name] = (params_w[p_name] - p_range[0]) / scale + normalize_flattened_hyperspace( + scaled_params_b, self._simplex[0].model._get_params(), self.params_range + ) + normalize_flattened_hyperspace( + scaled_params_g, self._simplex[1].model._get_params(), self.params_range + ) + normalize_flattened_hyperspace( + scaled_params_w, self._simplex[2].model._get_params(), self.params_range + ) max_dist = max( [ @@ -243,7 +294,7 @@ def _learn_not_converged(self, x, y): y_pred = wrap.model.predict_one(x) wrap.metric.update(y, y_pred) wrap.model.learn_one(x, y) - + # Keep the simplex ordered self._sort_simplex() @@ -275,7 +326,7 @@ def learn_one(self, x, y): self._learn_converged(x, y) else: self._learn_not_converged(x, y) - + return self @property @@ -292,10 +343,9 @@ def converged(self): return self._converged - class SSPTClassifier(SSPT, base.Classifier): """Single-pass Self Parameter Tuning Regressor. - + Parameters ---------- model @@ -312,6 +362,7 @@ class SSPTClassifier(SSPT, base.Classifier): [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021).Hyperparameter self-tuning for data streams. Information Fusion, 76, 75-86. """ + def __init__( self, model: base.Classifier, @@ -343,7 +394,7 @@ def predict_proba_one(self, x: dict): class SSPTRegressor(SSPT, base.Regressor): """Single-pass Self Parameter Tuning Regressor. - + Parameters ---------- model @@ -385,6 +436,7 @@ class SSPTRegressor(SSPT, base.Regressor): [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021).Hyperparameter self-tuning for data streams. Information Fusion, 76, 75-86. """ + def __init__( self, model: base.Regressor, From 1022b4a87a687c3a44b82480d3148cba6361a8f7 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Thu, 7 Jul 2022 18:47:25 +0100 Subject: [PATCH 06/50] sounds promising --- river_extra/model_selection/__init__.py | 4 +- river_extra/model_selection/sspt.py | 381 +++++++++++------------- 2 files changed, 168 insertions(+), 217 deletions(-) diff --git a/river_extra/model_selection/__init__.py b/river_extra/model_selection/__init__.py index 1423b59..43af3a6 100644 --- a/river_extra/model_selection/__init__.py +++ b/river_extra/model_selection/__init__.py @@ -1,3 +1,3 @@ -from .sspt import SSPTClassifier, SSPTRegressor +from .sspt import SSPT -__all__ = ["SSPTClassifier", "SSPTRegressor"] +__all__ = ["SSPT"] diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 308df61..653aca7 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -1,14 +1,14 @@ -import abc import collections import copy +import functools import math import random -import types import typing -from river import base, drift, metrics, utils +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics, utils -ModelWrapper = collections.namedtuple("ModelWrapper", "model metric") +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") class SSPT(base.Estimator): @@ -19,19 +19,21 @@ class SSPT(base.Estimator): def __init__( self, - model, - metric, + estimator: base.Estimator, + metric: metrics.base.Metric, params_range: typing.Dict[str, typing.Tuple], - grace_period: int, - drift_detector: base.DriftDetector, - start: str, - convergence_sphere: float, - seed: int, + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + start: str = "warm", + convergence_sphere: float = 0.001, + seed: int = None, ): super().__init__() - self.model = model + self.estimator = estimator self.metric = metric self.params_range = params_range + self.drift_input = drift_input self.grace_period = grace_period self.drift_detector = drift_detector @@ -49,39 +51,54 @@ def __init__( self._converged = False self._rng = random.Random(self.seed) - self._best_model = None - self._simplex = self._create_simplex(model) + self._best_estimator = None + self._simplex = self._create_simplex(estimator) # Models expanded from the simplex self._expanded: typing.Optional[typing.Dict] = None # Meta-programming - self._bind_output_method() - - def _bind_output_method(self): - pass + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" def _random_config(self): - def gen_random(p_info): - if isinstance(p_info[-1], dict): - new_vals = {} - sub_class, sub_params = p_info - for p_name in sub_params: - new_vals[p_name] = gen_random(sub_params[p_name]) - return sub_class(**new_vals) - else: - p_type, p_range = p_info + def gen_random(p_data, e_data): + # Sub-component needs to be instantiated + if isinstance(e_data, tuple): + sub_class, sub_data = e_data + sub_config = {} + + for sub_param, sub_info in p_data.items(): + sub_config[sub_param] = gen_random(sub_info, sub_data[sub_param]) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(p_data, tuple): + p_type, p_range = p_data if p_type == int: return self._rng.randint(p_range[0], p_range[1]) elif p_type == float: return self._rng.uniform(p_range[0], p_range[1]) - config = {} - for p_name, p_info in self.params_range.items(): - config[p_name] = gen_random(p_info) + # The sub-parameters need to be expanded + config = {} + for p_name, p_info in p_data.items(): + e_info = e_data[p_name] + sub_config = {} + for sub_name, sub_info in p_info.items(): + sub_config[sub_name] = gen_random(sub_info, e_info[sub_name]) + config[p_name] = sub_config + return config - return config + return gen_random(self.params_range, self.estimator._get_params()) def _create_simplex(self, model) -> typing.List: # The simplex is divided in: @@ -91,16 +108,16 @@ def _create_simplex(self, model) -> typing.List: simplex = [None] * 3 simplex[0] = ModelWrapper( - self.model.clone(self._random_config()), self.metric.clone() + self.estimator.clone(self._random_config()), self.metric.clone() ) simplex[2] = ModelWrapper( - self.model.clone(self._random_config()), self.metric.clone() + self.estimator.clone(self._random_config()), self.metric.clone() ) if self.start == self._START_RANDOM: # The intermediate 'good' model is defined randomly simplex[1] = ModelWrapper( - self.model.clone(self._random_config()), self.metric.clone() + self.estimator.clone(self._random_config()), self.metric.clone() ) elif self.start == self._START_WARM: # The intermediate 'good' model is defined randomly @@ -115,23 +132,21 @@ def _sort_simplex(self): else: self._simplex.sort(key=lambda mw: mw.metric.get()) - def _nelder_mead_expansion(self) -> typing.Dict: - """Create expanded models given the simplex models.""" + def _gen_new_estimator(self, e1, e2, func): + """Generate new configuration given two estimators and a combination function.""" - def apply_operator(param1, param2, func, p_info): - if isinstance(p_info[1], dict): + def apply_operator(param1, param2, p_info, func): + if isinstance(param1, tuple): sub_class, sub_params1 = param1 _, sub_params2 = param2 - sub_info = p_info[1] - new_params = {} - for sp_name in sub_params1: - sp_info = sub_info[sp_name] - # Recursive call to deal with nested hiperparameters - new_params[sp_name] = apply_operator( - sub_params1[sp_name], sub_params2[sp_name], func, sp_info + + sub_config = {} + for sub_param, sub_info in p_info.items(): + sub_config[sub_param] = apply_operator( + sub_params1[sub_param], sub_params2[sub_param], sub_info, func ) - return sub_class(**new_params) - else: + return sub_class(**sub_config) + if isinstance(p_info, tuple): p_type, p_range = p_info new_val = func(param1, param2) @@ -142,46 +157,57 @@ def apply_operator(param1, param2, func, p_info): new_val = p_range[1] new_val = round(new_val, 0) if p_type == int else new_val - return new_val - def gen_new(m1, m2, func): - new_config = {} - m1_params = m1.model._get_params() - m2_params = m2.model._get_params() + # The sub-parameters need to be expanded + config = {} + for p_name, inner_p_info in p_info.items(): + sub_param1 = param1[p_name] + sub_param2 = param2[p_name] + + sub_config = {} + for sub_name, sub_info in inner_p_info.items(): + sub_config[sub_name] = apply_operator( + sub_param1[sub_name], sub_param2[sub_name], sub_info, func + ) + config[p_name] = sub_config + return config - for p_name, p_info in self.params_range.items(): - new_config[p_name] = apply_operator( - m1_params[p_name], m2_params[p_name], func, p_info - ) + e1_params = e1.estimator._get_params() + e2_params = e2.estimator._get_params() - # Modify the current best contender with the new hyperparameter values - new = ModelWrapper( - copy.deepcopy(self._simplex[0].model), self.metric.clone() - ) + new_config = apply_operator(e1_params, e2_params, self.params_range, func) + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(self._simplex[0].estimator), self.metric.clone() + ) + new.estimator.mutate(new_config) - return new + return new + + def _nelder_mead_expansion(self) -> typing.Dict: + """Create expanded models given the simplex models.""" expanded = {} # Midpoint between 'best' and 'good' - expanded["midpoint"] = gen_new( + expanded["midpoint"] = self._gen_new_estimator( self._simplex[0], self._simplex[1], lambda h1, h2: (h1 + h2) / 2 ) # Reflection of 'midpoint' towards 'worst' - expanded["reflection"] = gen_new( + expanded["reflection"] = self._gen_new_estimator( expanded["midpoint"], self._simplex[2], lambda h1, h2: 2 * h1 - h2 ) # Expand the 'reflection' point - expanded["expansion"] = gen_new( + expanded["expansion"] = self._gen_new_estimator( expanded["reflection"], expanded["midpoint"], lambda h1, h2: 2 * h1 - h2 ) # Shrink 'best' and 'worst' - expanded["shrink"] = gen_new( + expanded["shrink"] = self._gen_new_estimator( self._simplex[0], self._simplex[2], lambda h1, h2: (h1 + h2) / 2 ) # Contraction of 'midpoint' and 'worst' - expanded["contraction"] = gen_new( + expanded["contraction"] = self._gen_new_estimator( expanded["midpoint"], self._simplex[2], lambda h1, h2: (h1 + h2) / 2 ) @@ -223,30 +249,38 @@ def _nelder_mead_operators(self): def _models_converged(self) -> bool: # Normalize params to ensure they contribute equally to the stopping criterion def normalize_flattened_hyperspace(scaled, orig, info, prefix=""): + if isinstance(orig, tuple): + _, sub_orig = orig + for sub_param, sub_info in info.items(): + prefix_ = prefix + "__" + sub_param + normalize_flattened_hyperspace( + scaled, sub_orig[sub_param], sub_info, prefix_ + ) + return + + if isinstance(info, tuple): + _, p_range = info + interval = p_range[1] - p_range[0] + scaled[prefix] = (orig - p_range[0]) / interval + return + for p_name, p_info in info.items(): - prefix_ = prefix + p_name - if isinstance(p_info[-1], dict): - sub_orig = orig[p_name][1] - sub_info = p_info[1] - prefix_ += "__" - normalize_flattened_hyperspace(scaled, sub_orig, sub_info, prefix_) - else: - _, p_range = p_info - interval = p_range[1] - p_range[0] - scaled[prefix_] = (orig[p_name] - p_range[0]) / interval + sub_orig = orig[p_name] + prefix_ = prefix + "__" + p_name if len(prefix) > 0 else p_name + normalize_flattened_hyperspace(scaled, sub_orig, p_info, prefix_) # 1. Simplex in sphere scaled_params_b = {} scaled_params_g = {} scaled_params_w = {} normalize_flattened_hyperspace( - scaled_params_b, self._simplex[0].model._get_params(), self.params_range + scaled_params_b, self._simplex[0].estimator._get_params(), self.params_range ) normalize_flattened_hyperspace( - scaled_params_g, self._simplex[1].model._get_params(), self.params_range + scaled_params_g, self._simplex[1].estimator._get_params(), self.params_range ) normalize_flattened_hyperspace( - scaled_params_w, self._simplex[2].model._get_params(), self.params_range + scaled_params_w, self._simplex[2].estimator._get_params(), self.params_range ) max_dist = max( @@ -267,33 +301,31 @@ def normalize_flattened_hyperspace(scaled, orig, info, prefix=""): return False - @abc.abstractmethod - def _drift_input(self, y_true, y_pred) -> typing.Union[int, float]: - pass - def _learn_converged(self, x, y): - y_pred = self._best_model.predict_one(x) + scorer = getattr(self._best_estimator, self._scorer_name) + y_pred = scorer(x) - input = self._drift_input(y, y_pred) + input = self.drift_input(y, y_pred) self.drift_detector.update(input) # We need to start the optimization process from scratch if self.drift_detector.drift_detected: self._n = 0 self._converged = False - self._simplex = self._create_simplex(self._best_model) + self._simplex = self._create_simplex(self._best_estimator) # There is no proven best model right now - self._best_model = None + self._best_estimator = None return - self._best_model.learn_one(x, y) + self._best_estimator.learn_one(x, y) def _learn_not_converged(self, x, y): for wrap in self._simplex: - y_pred = wrap.model.predict_one(x) + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) wrap.metric.update(y, y_pred) - wrap.model.learn_one(x, y) + wrap.estimator.learn_one(x, y) # Keep the simplex ordered self._sort_simplex() @@ -302,9 +334,10 @@ def _learn_not_converged(self, x, y): self._expanded = self._nelder_mead_expansion() for wrap in self._expanded.values(): - y_pred = wrap.model.predict_one(x) + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) wrap.metric.update(y, y_pred) - wrap.model.learn_one(x, y) + wrap.estimator.learn_one(x, y) if self._n == self.grace_period: self._n = 0 @@ -317,7 +350,7 @@ def _learn_not_converged(self, x, y): if self._models_converged: self._converged = True - self._best_model = self._simplex[0].model + self._best_estimator = self._simplex[0].estimator def learn_one(self, x, y): self._n += 1 @@ -330,137 +363,55 @@ def learn_one(self, x, y): return self @property - def best_model(self): + def best(self): if not self._converged: # Lazy selection of the best model self._sort_simplex() - return self._simplex[0].model + return self._simplex[0].estimator - return self._best_model + return self._best_estimator @property def converged(self): return self._converged + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) -class SSPTClassifier(SSPT, base.Classifier): - """Single-pass Self Parameter Tuning Regressor. - - Parameters - ---------- - model - metric - params_range - grace_period - drift_detector - start - convergence_sphere - seed - - References - ---------- - [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021).Hyperparameter self-tuning - for data streams. Information Fusion, 76, 75-86. - """ - - def __init__( - self, - model: base.Classifier, - metric: metrics.base.ClassificationMetric, - params_range: typing.Dict[str, typing.Tuple], - grace_period: int = 500, - drift_detector: base.DriftDetector = drift.ADWIN(), - start: str = "warm", - convergence_sphere: float = 0.0001, - seed: int = None, - ): - super().__init__( - model, - metric, - params_range, - grace_period, - drift_detector, - start, - convergence_sphere, - seed, - ) - - def _drift_input(self, y_true, y_pred): - return 0 if y_true == y_pred else 1 - - def predict_proba_one(self, x: dict): - return self.best_model.predict_proba_one(x) - - -class SSPTRegressor(SSPT, base.Regressor): - """Single-pass Self Parameter Tuning Regressor. - - Parameters - ---------- - model - metric - params_range - grace_period - drift_detector - start - convergence_sphere - seed - - Examples - -------- - >>> from river import datasets - >>> from river import linear_model - >>> from river import metrics - >>> from river import preprocessing - >>> from river_extra import model_selection - - >>> dataset = datasets.synth.Friedman(seed=42).take(2000) - >>> reg = preprocessing.StandardScaler() | model_selection.SSPTRegressor( - model=linear_model.LinearRegressor(), - metric=metrics.RMSE(), - params_range={ - "l2": (float, (0.0, 0.5)) - } - ) - >>> metric = metrics.RMSE() - - >>> for x, y in dataset: - ... y_pred = reg.predict_one(x) - ... metric.update(y, y_pred) - ... reg.learn_one(x, y) - - >>> metric - - References - ---------- - [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021).Hyperparameter self-tuning - for data streams. Information Fusion, 76, 75-86. - """ - - def __init__( - self, - model: base.Regressor, - metric: metrics.base.RegressionMetric, - params_range: typing.Dict[str, typing.Tuple], - grace_period: int = 500, - drift_detector: base.DriftDetector = drift.ADWIN(), - start: str = "warm", - convergence_sphere: float = 0.0001, - seed: int = None, - ): - super().__init__( - model, - metric, - params_range, - grace_period, - drift_detector, - start, - convergence_sphere, - seed, - ) + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) - def _drift_input(self, y_true, y_pred): - return abs(y_true - y_pred) + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) - def predict_one(self, x: dict): - return self.best_model.predict_one(x) + def debug_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) From 4a151c31294b8a8b431491fb05b962a1b09c9951 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Thu, 7 Jul 2022 18:51:40 +0100 Subject: [PATCH 07/50] add reference --- river_extra/model_selection/sspt.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 653aca7..01b9da0 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -12,7 +12,26 @@ class SSPT(base.Estimator): - """Single-pass Self Parameter Tuning""" + """Single-pass Self Parameter Tuning + + Parameters + ---------- + estimator + metric + params_range + drift_input + grace_period + drift_detector + start + convergence_sphere + seed + + References + ---------- + [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning + for data streams. Information Fusion, 76, 75-86. + + """ _START_RANDOM = "random" _START_WARM = "warm" From 1045042973b771e808a5916d2d83c18a1a8327a0 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Thu, 28 Jul 2022 15:38:25 +0100 Subject: [PATCH 08/50] 1commit teste --- .idea/.gitignore | 0 .../inspectionProfiles/profiles_settings.xml | 6 +++ .idea/misc.xml | 4 ++ .idea/modules.xml | 8 +++ .idea/river-extra.iml | 14 +++++ .idea/vcs.xml | 6 +++ .idea/workspace.xml | 54 +++++++++++++++++++ river_extra/model_selection/sspt.py | 1 + 8 files changed, 93 insertions(+) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/river-extra.iml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..fae03c6 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..73a810c --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/river-extra.iml b/.idea/river-extra.iml new file mode 100644 index 0000000..b35cc76 --- /dev/null +++ b/.idea/river-extra.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..eb8ad03 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + { + "keyToString": { + "RunOnceActivity.OpenProjectViewOnStart": "true", + "RunOnceActivity.ShowReadmeOnStart": "true" + } +} + + + + + 1658417902827 + + + + + + + \ No newline at end of file diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 01b9da0..66c5fb8 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -47,6 +47,7 @@ def __init__( start: str = "warm", convergence_sphere: float = 0.001, seed: int = None, + # ): super().__init__() self.estimator = estimator From 9ed652a384eaf489cc7afa0692ed0d5d352dc3bf Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Fri, 26 Aug 2022 09:58:18 +0100 Subject: [PATCH 09/50] V0.1 Added 2 Stopping Criteria Added Contraction2 Fixed Convergence --- .idea/workspace.xml | 56 ++++++++++++++- river_extra/model_selection/sspt.py | 102 ++++++++++++++++++---------- river_extra/model_selection/test.py | 45 ++++++++++++ 3 files changed, 165 insertions(+), 38 deletions(-) create mode 100644 river_extra/model_selection/test.py diff --git a/.idea/workspace.xml b/.idea/workspace.xml index eb8ad03..6909df1 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,12 +1,22 @@ - + + + + + + + \ No newline at end of file diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 66c5fb8..95350e6 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -4,6 +4,7 @@ import math import random import typing +import numpy as np # TODO use lazy imports where needed from river import anomaly, base, compose, drift, metrics, utils @@ -47,6 +48,7 @@ def __init__( start: str = "warm", convergence_sphere: float = 0.001, seed: int = None, + verbose: bool = False, # ): super().__init__() @@ -57,6 +59,7 @@ def __init__( self.grace_period = grace_period self.drift_detector = drift_detector + self.verbose = verbose if start not in {self._START_RANDOM, self._START_WARM}: raise ValueError( @@ -227,9 +230,13 @@ def _nelder_mead_expansion(self) -> typing.Dict: self._simplex[0], self._simplex[2], lambda h1, h2: (h1 + h2) / 2 ) # Contraction of 'midpoint' and 'worst' - expanded["contraction"] = self._gen_new_estimator( + expanded["contraction1"] = self._gen_new_estimator( expanded["midpoint"], self._simplex[2], lambda h1, h2: (h1 + h2) / 2 ) + # Contraction of 'midpoint' and 'reflection' + expanded["contraction2"] = self._gen_new_estimator( + expanded["midpoint"], expanded["reflection"], lambda h1, h2: (h1 + h2) / 2 + ) return expanded @@ -238,7 +245,12 @@ def _nelder_mead_operators(self): g = self._simplex[1].metric w = self._simplex[2].metric r = self._expanded["reflection"].metric - + c1 = self._expanded["contraction1"].metric + c2 = self._expanded["contraction2"].metric + if c1.is_better_than(c2): + self._expanded["contraction"] = self._expanded["contraction1"] + else: + self._expanded["contraction"] = self._expanded["contraction2"] if r.is_better_than(g): if b.is_better_than(r): self._simplex[2] = self._expanded["reflection"] @@ -256,50 +268,48 @@ def _nelder_mead_operators(self): if c.is_better_than(w): self._simplex[2] = self._expanded["contraction"] else: - s = self._expanded["shrink"].metric - if s.is_better_than(w): - self._simplex[2] = self._expanded["shrink"] - m = self._expanded["midpoint"].metric - if m.is_better_than(g): - self._simplex[1] = self._expanded["midpoint"] + self._simplex[2] = self._expanded["shrink"] + self._simplex[1] = self._expanded["midpoint"] self._sort_simplex() + def _normalize_flattened_hyperspace(self, scaled, orig, info, prefix=""): + if isinstance(orig, tuple): + _, sub_orig = orig + for sub_param, sub_info in info.items(): + prefix_ = prefix + "__" + sub_param + self._normalize_flattened_hyperspace( + scaled, sub_orig[sub_param], sub_info, prefix_ + ) + return + + if isinstance(info, tuple): + _, p_range = info + interval = p_range[1] - p_range[0] + scaled[prefix] = (orig - p_range[0]) / interval + return + + for p_name, p_info in info.items(): + sub_orig = orig[p_name] + prefix_ = prefix + "__" + p_name if len(prefix) > 0 else p_name + self._normalize_flattened_hyperspace(scaled, sub_orig, p_info, prefix_) + @property def _models_converged(self) -> bool: # Normalize params to ensure they contribute equally to the stopping criterion - def normalize_flattened_hyperspace(scaled, orig, info, prefix=""): - if isinstance(orig, tuple): - _, sub_orig = orig - for sub_param, sub_info in info.items(): - prefix_ = prefix + "__" + sub_param - normalize_flattened_hyperspace( - scaled, sub_orig[sub_param], sub_info, prefix_ - ) - return - - if isinstance(info, tuple): - _, p_range = info - interval = p_range[1] - p_range[0] - scaled[prefix] = (orig - p_range[0]) / interval - return - for p_name, p_info in info.items(): - sub_orig = orig[p_name] - prefix_ = prefix + "__" + p_name if len(prefix) > 0 else p_name - normalize_flattened_hyperspace(scaled, sub_orig, p_info, prefix_) # 1. Simplex in sphere scaled_params_b = {} scaled_params_g = {} scaled_params_w = {} - normalize_flattened_hyperspace( + self._normalize_flattened_hyperspace( scaled_params_b, self._simplex[0].estimator._get_params(), self.params_range ) - normalize_flattened_hyperspace( + self._normalize_flattened_hyperspace( scaled_params_g, self._simplex[1].estimator._get_params(), self.params_range ) - normalize_flattened_hyperspace( + self._normalize_flattened_hyperspace( scaled_params_w, self._simplex[2].estimator._get_params(), self.params_range ) @@ -311,14 +321,18 @@ def normalize_flattened_hyperspace(scaled, orig, info, prefix=""): ] ) + Listv = [list(scaled_params_b.values()),list(scaled_params_g.values()),list(scaled_params_w.values())] + + vectors = np.array(Listv) + new_centroid = dict(zip(scaled_params_b.keys(), np.mean(vectors, axis=0))) + centroid_distance = utils.math.minkowski_distance(self.old_centroid, new_centroid, p=2) + self.old_centroid = new_centroid ndim = len(self.params_range) r_sphere = max_dist * math.sqrt((ndim / (2 * (ndim + 1)))) - if r_sphere < self.convergence_sphere: + if r_sphere < self.convergence_sphere or centroid_distance == 0: return True - # TODO? 2. Simplex did not change - return False def _learn_converged(self, x, y): @@ -361,6 +375,22 @@ def _learn_not_converged(self, x, y): if self._n == self.grace_period: self._n = 0 + # 1. Simplex in sphere + scaled_params_b = {} + scaled_params_g = {} + scaled_params_w = {} + self._normalize_flattened_hyperspace( + scaled_params_b, self._simplex[0].estimator._get_params(), self.params_range + ) + self._normalize_flattened_hyperspace( + scaled_params_g, self._simplex[1].estimator._get_params(), self.params_range + ) + self._normalize_flattened_hyperspace( + scaled_params_w, self._simplex[2].estimator._get_params(), self.params_range + ) + Listv = [list(scaled_params_b.values()), list(scaled_params_g.values()), list(scaled_params_w.values())] + vectors = np.array(Listv) + self.old_centroid = dict(zip(scaled_params_b.keys(), np.mean(vectors, axis=0))) # Update the simplex models using Nelder-Mead heuristics self._nelder_mead_operators() @@ -368,9 +398,9 @@ def _learn_not_converged(self, x, y): # Discard expanded models self._expanded = None - if self._models_converged: - self._converged = True - self._best_estimator = self._simplex[0].estimator + if self._models_converged: + self._converged = True + self._best_estimator = self._simplex[0].estimator def learn_one(self, x, y): self._n += 1 diff --git a/river_extra/model_selection/test.py b/river_extra/model_selection/test.py new file mode 100644 index 0000000..0d98f99 --- /dev/null +++ b/river_extra/model_selection/test.py @@ -0,0 +1,45 @@ +from river import datasets +from river import linear_model +from river import metrics +from river import preprocessing +from river_extra import model_selection + +dataset = datasets.synth.Friedman(seed=42).take(50000) + +sspt = model_selection.SSPT( + estimator=preprocessing.AdaptiveStandardScaler() | linear_model.LinearRegression(), + metric=metrics.RMSE(), + grace_period=500, + params_range={ + "AdaptiveStandardScaler": { + "alpha": (float, (0.1, 0.9)) + }, + "LinearRegression": { + "l2": (float, (0.0, 0.2)), + "optimizer": { + "lr": {"learning_rate": (float, (0.0091, 0.5))} + }, + "intercept_lr": {"learning_rate": (float, (0.0001, 0.5))} + } + }, + start="random", + drift_input=lambda yt, yp: abs(yt - yp), + convergence_sphere=0.00001, + seed=42 +) +metric = metrics.RMSE() +first_print = True + +for i, (x, y) in enumerate(dataset): + y_pred = sspt.predict_one(x) + metric.update(y, y_pred) + sspt.learn_one(x, y) + + if sspt.converged and first_print: + print("Converged at:", i) + first_print = False + +print("Total instances:", i + 1) +print(metric) +print("Best params:") +print(repr(sspt.best)) \ No newline at end of file From 23c5c3c77f67a38630b4f337bbbbe7e75a759afc Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Tue, 30 Aug 2022 16:53:33 +0100 Subject: [PATCH 10/50] V0.2 --- .idea/.gitignore | 0 .../inspectionProfiles/profiles_settings.xml | 6 - .idea/misc.xml | 4 - .idea/modules.xml | 8 -- .idea/river-extra.iml | 14 --- .idea/vcs.xml | 6 - .idea/workspace.xml | 106 ------------------ 7 files changed, 144 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/river-extra.iml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index e69de29..0000000 diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index fae03c6..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 73a810c..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/river-extra.iml b/.idea/river-extra.iml deleted file mode 100644 index b35cc76..0000000 --- a/.idea/river-extra.iml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 6909df1..0000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - - - - - - - - - - - - - - - - - { - "keyToString": { - "RunOnceActivity.OpenProjectViewOnStart": "true", - "RunOnceActivity.ShowReadmeOnStart": "true", - "WebServerToolWindowFactoryState": "false", - "nodejs_package_manager_path": "npm", - "settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" - } -} - - - - - - - - - - - - - - - 1658417902827 - - - - - - - - - - - - - \ No newline at end of file From 9feded948e298a36df487f26ee8826c976ca79eb Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Tue, 30 Aug 2022 16:54:15 +0100 Subject: [PATCH 11/50] V0.2 --- .idea/.gitignore | 0 .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 4 + .idea/modules.xml | 8 ++ .idea/river-extra.iml | 14 +++ .idea/vcs.xml | 6 + .idea/workspace.xml | 106 ++++++++++++++++++ 7 files changed, 144 insertions(+) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/river-extra.iml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..fae03c6 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..73a810c --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/river-extra.iml b/.idea/river-extra.iml new file mode 100644 index 0000000..b35cc76 --- /dev/null +++ b/.idea/river-extra.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..92f31f1 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + { + "keyToString": { + "RunOnceActivity.OpenProjectViewOnStart": "true", + "RunOnceActivity.ShowReadmeOnStart": "true", + "WebServerToolWindowFactoryState": "false", + "nodejs_package_manager_path": "npm", + "settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" + } +} + + + + + + + + + + + + + + + 1658417902827 + + + + + + + + + + + + + \ No newline at end of file From 33b08ad878857f104ec5c4eeeac36a102347e695 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Tue, 30 Aug 2022 16:57:18 +0100 Subject: [PATCH 12/50] V0.3 --- .gitignore | 1 + .idea/workspace.xml | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4c6379b..44eccf2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea/ poetry.lock *.whl *.tar.gz diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 92f31f1..927aa5d 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,7 +1,8 @@ - + + @@ -99,6 +123,10 @@ + + + From 02b66842f5f47421aafddb0fd2b598a27323f303 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Tue, 30 Aug 2022 16:59:52 +0100 Subject: [PATCH 13/50] v0.4 --- .idea/.gitignore | 0 .../inspectionProfiles/profiles_settings.xml | 6 - .idea/misc.xml | 4 - .idea/modules.xml | 8 -- .idea/river-extra.iml | 14 -- .idea/vcs.xml | 6 - .idea/workspace.xml | 134 ------------------ 7 files changed, 172 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/river-extra.iml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index e69de29..0000000 diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index fae03c6..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 73a810c..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/river-extra.iml b/.idea/river-extra.iml deleted file mode 100644 index b35cc76..0000000 --- a/.idea/river-extra.iml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 927aa5d..0000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,134 +0,0 @@ - - - - - - - - - - - - - - - - - - - - { - "keyToString": { - "RunOnceActivity.OpenProjectViewOnStart": "true", - "RunOnceActivity.ShowReadmeOnStart": "true", - "WebServerToolWindowFactoryState": "false", - "nodejs_package_manager_path": "npm", - "settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" - } -} - - - - - - - - - - - - - - - 1658417902827 - - - 1661874813607 - - - 1661874855462 - - - 1661874907408 - - - - - - - - - - - - - - - \ No newline at end of file From 0ccd382f1c3bb0ec722a42365a931c51c849b005 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Tue, 6 Sep 2022 12:03:43 +0100 Subject: [PATCH 14/50] V0.5 --- .../model_selection/classification_test.py | 87 ++++++++++++++++++ .../model_selection/clustering_test.py | 76 ++++++++++++++++ .../model_selection/recommendation_test.py | 80 ++++++++++++++++ .../model_selection/regression_test.py | 91 +++++++++++++++++++ river_extra/model_selection/sspt.py | 16 +++- river_extra/model_selection/test.py | 45 --------- 6 files changed, 345 insertions(+), 50 deletions(-) create mode 100644 river_extra/model_selection/classification_test.py create mode 100644 river_extra/model_selection/clustering_test.py create mode 100644 river_extra/model_selection/recommendation_test.py create mode 100644 river_extra/model_selection/regression_test.py delete mode 100644 river_extra/model_selection/test.py diff --git a/river_extra/model_selection/classification_test.py b/river_extra/model_selection/classification_test.py new file mode 100644 index 0000000..4f355a3 --- /dev/null +++ b/river_extra/model_selection/classification_test.py @@ -0,0 +1,87 @@ +from river import datasets, utils, drift, tree +from river import linear_model +from river import metrics +from river import preprocessing +from river.datasets import synth + +from river_extra import model_selection +import matplotlib.pyplot as plt + +# Dataset +dataset = datasets.synth.ConceptDriftStream(stream=synth.SEA(seed=42, variant=0), + drift_stream=synth.SEA(seed=42, variant=1), + seed=1, position=5000, width=2).take(10000) + + +# Baseline - model and metric +baseline_metric = metrics.Accuracy() +baseline_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=100) +baseline_metric_plt=[] +baseline_rolling_metric_plt=[] +baseline = ( + preprocessing.AdaptiveStandardScaler() | + tree.HoeffdingTreeClassifier() +) + +# SSPT - model and metric +sspt_metric = metrics.Accuracy() +sspt_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=100) +sspt_metric_plt=[] +sspt_rolling_metric_plt=[] +sspt = model_selection.SSPT( + estimator=preprocessing.AdaptiveStandardScaler() | tree.HoeffdingTreeClassifier(), + metric=sspt_rolling_metric, + grace_period=100, + params_range={ + "AdaptiveStandardScaler": { + "alpha": (float, (0.25, 0.35)) + }, + "HoeffdingTreeClassifier": { + "delta": (float, (0.00001, 0.0001)), + "grace_period": (int, (100, 500)) + } + }, + start="random", + drift_input=lambda yt, yp: abs(yt - yp), + #drift_detector=drift.PageHinkley(), + convergence_sphere=0.000001, + seed=42 +) + +first_print = True + + + +for i, (x, y) in enumerate(dataset): + baseline_y_pred = baseline.predict_one(x) + baseline_metric.update(y, baseline_y_pred) + baseline_rolling_metric.update(y, baseline_y_pred) + baseline_metric_plt.append(baseline_metric.get()) + baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) + baseline.learn_one(x, y) + sspt_y_pred = sspt.predict_one(x) + sspt_metric.update(y, sspt_y_pred) + sspt_rolling_metric.update(y, sspt_y_pred) + sspt_metric_plt.append(sspt_metric.get()) + sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) + sspt.learn_one(x, y) + + if sspt.converged and first_print: + print("Converged at:", i) + first_print = False + +print("Total instances:", i + 1) +print(repr(baseline)) +print("Best params:") +print(repr(sspt.best)) +print("SSPT: ",sspt_metric) +print("Baseline: ",baseline_metric) + + +plt.plot(baseline_metric_plt[:10000], linestyle = 'dotted') +plt.plot(sspt_metric_plt[:10000]) +plt.show() + +plt.plot(baseline_rolling_metric_plt[:10000], linestyle = 'dotted') +plt.plot(sspt_rolling_metric_plt[:10000]) +plt.show() \ No newline at end of file diff --git a/river_extra/model_selection/clustering_test.py b/river_extra/model_selection/clustering_test.py new file mode 100644 index 0000000..863f544 --- /dev/null +++ b/river_extra/model_selection/clustering_test.py @@ -0,0 +1,76 @@ +from river import datasets, utils, drift, cluster +from river import linear_model +from river import metrics +from river import preprocessing +from river_extra import model_selection +import matplotlib.pyplot as plt + +# Dataset +dataset = datasets.Bananas() + + +# Baseline - model and metric +baseline_metric = metrics.Silhouette() +baseline_rolling_metric = utils.Rolling(metrics.Silhouette(), window_size=100) +baseline_metric_plt=[] +baseline_rolling_metric_plt=[] +baseline = cluster.CluStream() + +# SSPT - model and metric +sspt_metric = metrics.Silhouette() +sspt_rolling_metric = utils.Rolling(metrics.Silhouette(), window_size=100) +sspt_metric_plt=[] +sspt_rolling_metric_plt=[] +sspt = model_selection.SSPT( + estimator=cluster.CluStream(), + metric=sspt_rolling_metric, + grace_period=100, + params_range={ + "max_micro_clusters": (int, (1, 10)), + "n_macro_clusters": (int, (1, 10)), + "halflife": (float, (0.1, 0.5)) + }, + start="random", + drift_input=lambda yt, yp: abs(yt - yp), + #drift_detector=drift.PageHinkley(), + convergence_sphere=0.000001, + seed=42 +) + +first_print = True + + + +for i, (x, y) in enumerate(dataset): + baseline_y_pred = baseline.predict_one(x) + baseline_metric.update(y, baseline_y_pred, baseline.centers) + baseline_rolling_metric.update(y, baseline_y_pred, baseline.centers) + baseline_metric_plt.append(baseline_metric.get()) + baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) + baseline.learn_one(x, y) + sspt_y_pred = sspt.predict_one(x) + sspt_metric.update(y, sspt_y_pred) + sspt_rolling_metric.update(y, sspt_y_pred, sspt.estimator.centers) + sspt_metric_plt.append(sspt_metric.get()) + sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) + sspt.learn_one(x, y) + + if sspt.converged and first_print: + print("Converged at:", i) + first_print = False + +print("Total instances:", i + 1) +print(repr(baseline)) +print("Best params:") +print(repr(sspt.best)) +print("SSPT: ",sspt_metric) +print("Baseline: ",baseline_metric) + + +plt.plot(baseline_metric_plt[:10000], linestyle = 'dotted') +plt.plot(sspt_metric_plt[:10000]) +plt.show() + +plt.plot(baseline_rolling_metric_plt[:10000], linestyle = 'dotted') +plt.plot(sspt_rolling_metric_plt[:10000]) +plt.show() \ No newline at end of file diff --git a/river_extra/model_selection/recommendation_test.py b/river_extra/model_selection/recommendation_test.py new file mode 100644 index 0000000..b5e7664 --- /dev/null +++ b/river_extra/model_selection/recommendation_test.py @@ -0,0 +1,80 @@ +from river import datasets, utils, drift, reco, optim +from river import linear_model +from river import metrics +from river import preprocessing +from river_extra import model_selection +import matplotlib.pyplot as plt + +# Dataset +dataset = datasets.MovieLens100K() + + +# Baseline - model and metric +baseline_metric = metrics.RMSE() +baseline_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) +baseline_metric_plt=[] +baseline_rolling_metric_plt=[] +baseline = reco.BiasedMF() + +# SSPT - model and metric +sspt_metric = metrics.RMSE() +sspt_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) +sspt_metric_plt=[] +sspt_rolling_metric_plt=[] +sspt = model_selection.SSPT( + estimator=reco.BiasedMF(bias_optimizer=optim.SGD(), latent_optimizer=optim.SGD()), + metric=sspt_rolling_metric, + grace_period=100, + params_range={ + "latent_optimizer": { + "lr": {"learning_rate": (float, (0.01, 0.1))} + }, + "bias_optimizer": { + "lr": {"learning_rate": (float, (0.01, 0.1))} + } + + }, + start="random", + drift_input=lambda yt, yp: abs(yt - yp), + #drift_detector=drift.PageHinkley(), + convergence_sphere=0.000001, + seed=42 +) + +first_print = True + + + +for i, (x, y) in enumerate(dataset): + baseline_y_pred = baseline.predict_one(x) + baseline_metric.update(y, baseline_y_pred) + baseline_rolling_metric.update(y, baseline_y_pred) + baseline_metric_plt.append(baseline_metric.get()) + baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) + baseline.learn_one(x, y) + sspt_y_pred = sspt.predict_one(x) + sspt_metric.update(y, sspt_y_pred) + sspt_rolling_metric.update(y, sspt_y_pred) + sspt_metric_plt.append(sspt_metric.get()) + sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) + sspt.learn_one(x, y) + + if sspt.converged and first_print: + print("Converged at:", i) + first_print = False + +print("Total instances:", i + 1) +print(repr(baseline)) +print("Best params:") +print(repr(sspt.best)) +print("SSPT: ",sspt_metric) +print("Baseline: ",baseline_metric) + + +plt.plot(baseline_metric_plt[:10000], linestyle = 'dotted') +plt.plot(sspt_metric_plt[:10000]) +plt.show() + +plt.plot(baseline_rolling_metric_plt[:10000], linestyle = 'dotted') +plt.plot(sspt_rolling_metric_plt[:10000]) +plt.show() \ No newline at end of file diff --git a/river_extra/model_selection/regression_test.py b/river_extra/model_selection/regression_test.py new file mode 100644 index 0000000..eb615d9 --- /dev/null +++ b/river_extra/model_selection/regression_test.py @@ -0,0 +1,91 @@ +from river import datasets, utils, drift +from river import linear_model +from river import metrics +from river import preprocessing +from river_extra import model_selection +import matplotlib.pyplot as plt + +# Dataset +dataset = datasets.synth.FriedmanDrift( + drift_type='gra', + position=(7000, 9000), + seed=42 +).take(10000) + + +# Baseline - model and metric +baseline_metric = metrics.RMSE() +baseline_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) +baseline_metric_plt=[] +baseline_rolling_metric_plt=[] +baseline = ( + preprocessing.AdaptiveStandardScaler() | + linear_model.LinearRegression() +) + +# SSPT - model and metric +sspt_metric = metrics.RMSE() +sspt_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) +sspt_metric_plt=[] +sspt_rolling_metric_plt=[] +sspt = model_selection.SSPT( + estimator=preprocessing.AdaptiveStandardScaler() | linear_model.LinearRegression(), + metric=metrics.RMSE(), + grace_period=100, + params_range={ + "AdaptiveStandardScaler": { + "alpha": (float, (0.25, 0.35)) + }, + "LinearRegression": { + "l2": (float, (0.0, 0.0001)), + "optimizer": { + "lr": {"learning_rate": (float, (0.009, 0.011))} + }, + "intercept_lr": {"learning_rate": (float, (0.009, 0.011))} + } + }, + start="random", + drift_input=lambda yt, yp: abs(yt - yp), + drift_detector=drift.PageHinkley(), + convergence_sphere=0.000001, + seed=42 +) + +first_print = True + +metric = metrics.RMSE() + + +for i, (x, y) in enumerate(dataset): + baseline_y_pred = baseline.predict_one(x) + baseline_metric.update(y, baseline_y_pred) + baseline_rolling_metric.update(y, baseline_y_pred) + baseline_metric_plt.append(baseline_metric.get()) + baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) + baseline.learn_one(x, y) + sspt_y_pred = sspt.predict_one(x) + sspt_metric.update(y, sspt_y_pred) + sspt_rolling_metric.update(y, sspt_y_pred) + sspt_metric_plt.append(sspt_metric.get()) + sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) + sspt.learn_one(x, y) + + if sspt.converged and first_print: + print("Converged at:", i) + first_print = False + +print("Total instances:", i + 1) +print(repr(baseline)) +print("Best params:") +print(repr(sspt.best)) +print("SSPT: ",sspt_metric) +print("Baseline: ",baseline_metric) + + +plt.plot(baseline_metric_plt[:10000], linestyle = 'dotted') +plt.plot(sspt_metric_plt[:10000]) +plt.show() + +plt.plot(baseline_rolling_metric_plt[:10000], linestyle = 'dotted') +plt.plot(sspt_rolling_metric_plt[:10000]) +plt.show() \ No newline at end of file diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 95350e6..fb3dec8 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -140,11 +140,11 @@ def _create_simplex(self, model) -> typing.List: if self.start == self._START_RANDOM: # The intermediate 'good' model is defined randomly simplex[1] = ModelWrapper( - self.estimator.clone(self._random_config()), self.metric.clone() + self.estimator.clone(self._random_config()), self.metric.clone(include_attributes=True) ) elif self.start == self._START_WARM: # The intermediate 'good' model is defined randomly - simplex[1] = ModelWrapper(copy.deepcopy(model), self.metric.clone()) + simplex[1] = ModelWrapper(model.clone(include_attributes=True), self.metric.clone(include_attributes=True)) return simplex @@ -202,7 +202,7 @@ def apply_operator(param1, param2, p_info, func): new_config = apply_operator(e1_params, e2_params, self.params_range, func) # Modify the current best contender with the new hyperparameter values new = ModelWrapper( - copy.deepcopy(self._simplex[0].estimator), self.metric.clone() + copy.deepcopy(self._simplex[0].estimator), self.metric.clone(include_attributes=True) ) new.estimator.mutate(new_config) @@ -210,7 +210,8 @@ def apply_operator(param1, param2, p_info, func): def _nelder_mead_expansion(self) -> typing.Dict: """Create expanded models given the simplex models.""" - + #print('----------Simplex------------') + #print(self._simplex) expanded = {} # Midpoint between 'best' and 'good' expanded["midpoint"] = self._gen_new_estimator( @@ -237,7 +238,8 @@ def _nelder_mead_expansion(self) -> typing.Dict: expanded["contraction2"] = self._gen_new_estimator( expanded["midpoint"], expanded["reflection"], lambda h1, h2: (h1 + h2) / 2 ) - + #print('----------Expanded------------') + #print(expanded) return expanded def _nelder_mead_operators(self): @@ -388,6 +390,10 @@ def _learn_not_converged(self, x, y): self._normalize_flattened_hyperspace( scaled_params_w, self._simplex[2].estimator._get_params(), self.params_range ) + print('----------') + print('B:',list(scaled_params_b.values()),'Score:',self._simplex[0].metric) + print('G:', list(scaled_params_g.values()),'Score:',self._simplex[1].metric) + print('W:', list(scaled_params_w.values()),'Score:',self._simplex[2].metric) Listv = [list(scaled_params_b.values()), list(scaled_params_g.values()), list(scaled_params_w.values())] vectors = np.array(Listv) self.old_centroid = dict(zip(scaled_params_b.keys(), np.mean(vectors, axis=0))) diff --git a/river_extra/model_selection/test.py b/river_extra/model_selection/test.py deleted file mode 100644 index 0d98f99..0000000 --- a/river_extra/model_selection/test.py +++ /dev/null @@ -1,45 +0,0 @@ -from river import datasets -from river import linear_model -from river import metrics -from river import preprocessing -from river_extra import model_selection - -dataset = datasets.synth.Friedman(seed=42).take(50000) - -sspt = model_selection.SSPT( - estimator=preprocessing.AdaptiveStandardScaler() | linear_model.LinearRegression(), - metric=metrics.RMSE(), - grace_period=500, - params_range={ - "AdaptiveStandardScaler": { - "alpha": (float, (0.1, 0.9)) - }, - "LinearRegression": { - "l2": (float, (0.0, 0.2)), - "optimizer": { - "lr": {"learning_rate": (float, (0.0091, 0.5))} - }, - "intercept_lr": {"learning_rate": (float, (0.0001, 0.5))} - } - }, - start="random", - drift_input=lambda yt, yp: abs(yt - yp), - convergence_sphere=0.00001, - seed=42 -) -metric = metrics.RMSE() -first_print = True - -for i, (x, y) in enumerate(dataset): - y_pred = sspt.predict_one(x) - metric.update(y, y_pred) - sspt.learn_one(x, y) - - if sspt.converged and first_print: - print("Converged at:", i) - first_print = False - -print("Total instances:", i + 1) -print(metric) -print("Best params:") -print(repr(sspt.best)) \ No newline at end of file From d4b7d1f178fbaa0c4920a7e7f0ff851cf5f01384 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Tue, 6 Sep 2022 12:25:50 +0100 Subject: [PATCH 15/50] simplify simplex init --- .../model_selection/classification_test.py | 1 - .../model_selection/clustering_test.py | 1 - .../model_selection/recommendation_test.py | 1 - .../model_selection/regression_test.py | 1 - river_extra/model_selection/sspt.py | 34 +++++-------------- 5 files changed, 9 insertions(+), 29 deletions(-) diff --git a/river_extra/model_selection/classification_test.py b/river_extra/model_selection/classification_test.py index 4f355a3..5de0406 100644 --- a/river_extra/model_selection/classification_test.py +++ b/river_extra/model_selection/classification_test.py @@ -41,7 +41,6 @@ "grace_period": (int, (100, 500)) } }, - start="random", drift_input=lambda yt, yp: abs(yt - yp), #drift_detector=drift.PageHinkley(), convergence_sphere=0.000001, diff --git a/river_extra/model_selection/clustering_test.py b/river_extra/model_selection/clustering_test.py index 863f544..ebca5fb 100644 --- a/river_extra/model_selection/clustering_test.py +++ b/river_extra/model_selection/clustering_test.py @@ -30,7 +30,6 @@ "n_macro_clusters": (int, (1, 10)), "halflife": (float, (0.1, 0.5)) }, - start="random", drift_input=lambda yt, yp: abs(yt - yp), #drift_detector=drift.PageHinkley(), convergence_sphere=0.000001, diff --git a/river_extra/model_selection/recommendation_test.py b/river_extra/model_selection/recommendation_test.py index b5e7664..a12753e 100644 --- a/river_extra/model_selection/recommendation_test.py +++ b/river_extra/model_selection/recommendation_test.py @@ -34,7 +34,6 @@ } }, - start="random", drift_input=lambda yt, yp: abs(yt - yp), #drift_detector=drift.PageHinkley(), convergence_sphere=0.000001, diff --git a/river_extra/model_selection/regression_test.py b/river_extra/model_selection/regression_test.py index eb615d9..2eabe90 100644 --- a/river_extra/model_selection/regression_test.py +++ b/river_extra/model_selection/regression_test.py @@ -44,7 +44,6 @@ "intercept_lr": {"learning_rate": (float, (0.009, 0.011))} } }, - start="random", drift_input=lambda yt, yp: abs(yt - yp), drift_detector=drift.PageHinkley(), convergence_sphere=0.000001, diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index fb3dec8..a5733b6 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -1,6 +1,5 @@ import collections import copy -import functools import math import random import typing @@ -23,7 +22,6 @@ class SSPT(base.Estimator): drift_input grace_period drift_detector - start convergence_sphere seed @@ -45,11 +43,8 @@ def __init__( drift_input: typing.Callable[[float, float], float], grace_period: int = 500, drift_detector: base.DriftDetector = drift.ADWIN(), - start: str = "warm", convergence_sphere: float = 0.001, seed: int = None, - verbose: bool = False, - # ): super().__init__() self.estimator = estimator @@ -59,13 +54,6 @@ def __init__( self.grace_period = grace_period self.drift_detector = drift_detector - self.verbose = verbose - - if start not in {self._START_RANDOM, self._START_WARM}: - raise ValueError( - f"'start' must be either '{self._START_RANDOM}' or '{self._START_WARM}'." - ) - self.start = start self.convergence_sphere = convergence_sphere self.seed = seed @@ -131,20 +119,17 @@ def _create_simplex(self, model) -> typing.List: simplex = [None] * 3 simplex[0] = ModelWrapper( - self.estimator.clone(self._random_config()), self.metric.clone() + self.estimator.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True) ) - simplex[2] = ModelWrapper( - self.estimator.clone(self._random_config()), self.metric.clone() + simplex[1] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True) ) - - if self.start == self._START_RANDOM: - # The intermediate 'good' model is defined randomly - simplex[1] = ModelWrapper( - self.estimator.clone(self._random_config()), self.metric.clone(include_attributes=True) - ) - elif self.start == self._START_WARM: - # The intermediate 'good' model is defined randomly - simplex[1] = ModelWrapper(model.clone(include_attributes=True), self.metric.clone(include_attributes=True)) + simplex[2] = ModelWrapper( + self.estimator.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True) + ) return simplex @@ -300,7 +285,6 @@ def _normalize_flattened_hyperspace(self, scaled, orig, info, prefix=""): def _models_converged(self) -> bool: # Normalize params to ensure they contribute equally to the stopping criterion - # 1. Simplex in sphere scaled_params_b = {} scaled_params_g = {} From ba2fdcc44b062c54683a75c5eda18195c4fe1f49 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Tue, 6 Sep 2022 18:39:11 +0100 Subject: [PATCH 16/50] misc improvements --- .../model_selection/classification_test.py | 57 ++--- .../model_selection/clustering_test.py | 37 ++- .../model_selection/recommendation_test.py | 57 ++--- .../model_selection/regression_test.py | 49 ++-- river_extra/model_selection/sspt.py | 216 ++++++++++-------- 5 files changed, 215 insertions(+), 201 deletions(-) diff --git a/river_extra/model_selection/classification_test.py b/river_extra/model_selection/classification_test.py index 5de0406..b8358a1 100644 --- a/river_extra/model_selection/classification_test.py +++ b/river_extra/model_selection/classification_test.py @@ -1,56 +1,47 @@ -from river import datasets, utils, drift, tree -from river import linear_model -from river import metrics -from river import preprocessing +import matplotlib.pyplot as plt +from river import datasets, drift, linear_model, metrics, preprocessing, tree, utils from river.datasets import synth from river_extra import model_selection -import matplotlib.pyplot as plt # Dataset -dataset = datasets.synth.ConceptDriftStream(stream=synth.SEA(seed=42, variant=0), - drift_stream=synth.SEA(seed=42, variant=1), - seed=1, position=5000, width=2).take(10000) +dataset = datasets.synth.ConceptDriftStream( + stream=synth.SEA(seed=42, variant=0), + drift_stream=synth.SEA(seed=42, variant=1), + seed=1, + position=5000, + width=2, +).take(10000) # Baseline - model and metric baseline_metric = metrics.Accuracy() baseline_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=100) -baseline_metric_plt=[] -baseline_rolling_metric_plt=[] -baseline = ( - preprocessing.AdaptiveStandardScaler() | - tree.HoeffdingTreeClassifier() -) +baseline_metric_plt = [] +baseline_rolling_metric_plt = [] +baseline = preprocessing.AdaptiveStandardScaler() | tree.HoeffdingTreeClassifier() # SSPT - model and metric sspt_metric = metrics.Accuracy() sspt_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=100) -sspt_metric_plt=[] -sspt_rolling_metric_plt=[] +sspt_metric_plt = [] +sspt_rolling_metric_plt = [] sspt = model_selection.SSPT( - estimator=preprocessing.AdaptiveStandardScaler() | tree.HoeffdingTreeClassifier(), + estimator=tree.HoeffdingTreeClassifier(), metric=sspt_rolling_metric, grace_period=100, params_range={ - "AdaptiveStandardScaler": { - "alpha": (float, (0.25, 0.35)) - }, - "HoeffdingTreeClassifier": { - "delta": (float, (0.00001, 0.0001)), - "grace_period": (int, (100, 500)) - } + "delta": (float, (0.00001, 0.0001)), + "grace_period": (int, (100, 500)), }, - drift_input=lambda yt, yp: abs(yt - yp), - #drift_detector=drift.PageHinkley(), + drift_input=lambda yt, yp: 0 if yt == yp else 1, convergence_sphere=0.000001, - seed=42 + seed=42, ) first_print = True - for i, (x, y) in enumerate(dataset): baseline_y_pred = baseline.predict_one(x) baseline_metric.update(y, baseline_y_pred) @@ -73,14 +64,14 @@ print(repr(baseline)) print("Best params:") print(repr(sspt.best)) -print("SSPT: ",sspt_metric) -print("Baseline: ",baseline_metric) +print("SSPT: ", sspt_metric) +print("Baseline: ", baseline_metric) -plt.plot(baseline_metric_plt[:10000], linestyle = 'dotted') +plt.plot(baseline_metric_plt[:10000], linestyle="dotted") plt.plot(sspt_metric_plt[:10000]) plt.show() -plt.plot(baseline_rolling_metric_plt[:10000], linestyle = 'dotted') +plt.plot(baseline_rolling_metric_plt[:10000], linestyle="dotted") plt.plot(sspt_rolling_metric_plt[:10000]) -plt.show() \ No newline at end of file +plt.show() diff --git a/river_extra/model_selection/clustering_test.py b/river_extra/model_selection/clustering_test.py index ebca5fb..e30c878 100644 --- a/river_extra/model_selection/clustering_test.py +++ b/river_extra/model_selection/clustering_test.py @@ -1,9 +1,7 @@ -from river import datasets, utils, drift, cluster -from river import linear_model -from river import metrics -from river import preprocessing -from river_extra import model_selection import matplotlib.pyplot as plt +from river import cluster, datasets, drift, linear_model, metrics, preprocessing, utils + +from river_extra import model_selection # Dataset dataset = datasets.Bananas() @@ -12,34 +10,33 @@ # Baseline - model and metric baseline_metric = metrics.Silhouette() baseline_rolling_metric = utils.Rolling(metrics.Silhouette(), window_size=100) -baseline_metric_plt=[] -baseline_rolling_metric_plt=[] +baseline_metric_plt = [] +baseline_rolling_metric_plt = [] baseline = cluster.CluStream() # SSPT - model and metric sspt_metric = metrics.Silhouette() sspt_rolling_metric = utils.Rolling(metrics.Silhouette(), window_size=100) -sspt_metric_plt=[] -sspt_rolling_metric_plt=[] +sspt_metric_plt = [] +sspt_rolling_metric_plt = [] sspt = model_selection.SSPT( estimator=cluster.CluStream(), metric=sspt_rolling_metric, grace_period=100, params_range={ - "max_micro_clusters": (int, (1, 10)), - "n_macro_clusters": (int, (1, 10)), - "halflife": (float, (0.1, 0.5)) + "max_micro_clusters": (int, (1, 10)), + "n_macro_clusters": (int, (1, 10)), + "halflife": (float, (0.1, 0.5)), }, drift_input=lambda yt, yp: abs(yt - yp), - #drift_detector=drift.PageHinkley(), + # drift_detector=drift.PageHinkley(), convergence_sphere=0.000001, - seed=42 + seed=42, ) first_print = True - for i, (x, y) in enumerate(dataset): baseline_y_pred = baseline.predict_one(x) baseline_metric.update(y, baseline_y_pred, baseline.centers) @@ -62,14 +59,14 @@ print(repr(baseline)) print("Best params:") print(repr(sspt.best)) -print("SSPT: ",sspt_metric) -print("Baseline: ",baseline_metric) +print("SSPT: ", sspt_metric) +print("Baseline: ", baseline_metric) -plt.plot(baseline_metric_plt[:10000], linestyle = 'dotted') +plt.plot(baseline_metric_plt[:10000], linestyle="dotted") plt.plot(sspt_metric_plt[:10000]) plt.show() -plt.plot(baseline_rolling_metric_plt[:10000], linestyle = 'dotted') +plt.plot(baseline_rolling_metric_plt[:10000], linestyle="dotted") plt.plot(sspt_rolling_metric_plt[:10000]) -plt.show() \ No newline at end of file +plt.show() diff --git a/river_extra/model_selection/recommendation_test.py b/river_extra/model_selection/recommendation_test.py index a12753e..ad64c47 100644 --- a/river_extra/model_selection/recommendation_test.py +++ b/river_extra/model_selection/recommendation_test.py @@ -1,9 +1,16 @@ -from river import datasets, utils, drift, reco, optim -from river import linear_model -from river import metrics -from river import preprocessing -from river_extra import model_selection import matplotlib.pyplot as plt +from river import ( + datasets, + drift, + linear_model, + metrics, + optim, + preprocessing, + reco, + utils, +) + +from river_extra import model_selection # Dataset dataset = datasets.MovieLens100K() @@ -12,51 +19,45 @@ # Baseline - model and metric baseline_metric = metrics.RMSE() baseline_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) -baseline_metric_plt=[] -baseline_rolling_metric_plt=[] +baseline_metric_plt = [] +baseline_rolling_metric_plt = [] baseline = reco.BiasedMF() # SSPT - model and metric sspt_metric = metrics.RMSE() sspt_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) -sspt_metric_plt=[] -sspt_rolling_metric_plt=[] +sspt_metric_plt = [] +sspt_rolling_metric_plt = [] sspt = model_selection.SSPT( estimator=reco.BiasedMF(bias_optimizer=optim.SGD(), latent_optimizer=optim.SGD()), metric=sspt_rolling_metric, grace_period=100, params_range={ - "latent_optimizer": { - "lr": {"learning_rate": (float, (0.01, 0.1))} - }, - "bias_optimizer": { - "lr": {"learning_rate": (float, (0.01, 0.1))} - } - + "latent_optimizer": {"lr": {"learning_rate": (float, (0.01, 0.1))}}, + "bias_optimizer": {"lr": {"learning_rate": (float, (0.01, 0.1))}}, }, drift_input=lambda yt, yp: abs(yt - yp), - #drift_detector=drift.PageHinkley(), + # drift_detector=drift.PageHinkley(), convergence_sphere=0.000001, - seed=42 + seed=42, ) first_print = True - for i, (x, y) in enumerate(dataset): - baseline_y_pred = baseline.predict_one(x) + baseline_y_pred = baseline.predict_one(user=x["user"], item=x["item"]) baseline_metric.update(y, baseline_y_pred) baseline_rolling_metric.update(y, baseline_y_pred) baseline_metric_plt.append(baseline_metric.get()) baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) - baseline.learn_one(x, y) - sspt_y_pred = sspt.predict_one(x) + baseline.learn_one(user=x["user"], item=x["item"], y=y) + sspt_y_pred = sspt.predict_one(user=x["user"], item=x["item"]) sspt_metric.update(y, sspt_y_pred) sspt_rolling_metric.update(y, sspt_y_pred) sspt_metric_plt.append(sspt_metric.get()) sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) - sspt.learn_one(x, y) + sspt.learn_one(user=x["user"], item=x["item"], y=y) if sspt.converged and first_print: print("Converged at:", i) @@ -66,14 +67,14 @@ print(repr(baseline)) print("Best params:") print(repr(sspt.best)) -print("SSPT: ",sspt_metric) -print("Baseline: ",baseline_metric) +print("SSPT: ", sspt_metric) +print("Baseline: ", baseline_metric) -plt.plot(baseline_metric_plt[:10000], linestyle = 'dotted') +plt.plot(baseline_metric_plt[:10000], linestyle="dotted") plt.plot(sspt_metric_plt[:10000]) plt.show() -plt.plot(baseline_rolling_metric_plt[:10000], linestyle = 'dotted') +plt.plot(baseline_rolling_metric_plt[:10000], linestyle="dotted") plt.plot(sspt_rolling_metric_plt[:10000]) -plt.show() \ No newline at end of file +plt.show() diff --git a/river_extra/model_selection/regression_test.py b/river_extra/model_selection/regression_test.py index 2eabe90..31ad93b 100644 --- a/river_extra/model_selection/regression_test.py +++ b/river_extra/model_selection/regression_test.py @@ -1,53 +1,42 @@ -from river import datasets, utils, drift -from river import linear_model -from river import metrics -from river import preprocessing -from river_extra import model_selection import matplotlib.pyplot as plt +from river import datasets, drift, linear_model, metrics, preprocessing, utils + +from river_extra import model_selection # Dataset dataset = datasets.synth.FriedmanDrift( - drift_type='gra', - position=(7000, 9000), - seed=42 + drift_type="gra", position=(7000, 9000), seed=42 ).take(10000) # Baseline - model and metric baseline_metric = metrics.RMSE() baseline_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) -baseline_metric_plt=[] -baseline_rolling_metric_plt=[] -baseline = ( - preprocessing.AdaptiveStandardScaler() | - linear_model.LinearRegression() -) +baseline_metric_plt = [] +baseline_rolling_metric_plt = [] +baseline = preprocessing.AdaptiveStandardScaler() | linear_model.LinearRegression() # SSPT - model and metric sspt_metric = metrics.RMSE() sspt_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) -sspt_metric_plt=[] -sspt_rolling_metric_plt=[] +sspt_metric_plt = [] +sspt_rolling_metric_plt = [] sspt = model_selection.SSPT( estimator=preprocessing.AdaptiveStandardScaler() | linear_model.LinearRegression(), metric=metrics.RMSE(), grace_period=100, params_range={ - "AdaptiveStandardScaler": { - "alpha": (float, (0.25, 0.35)) - }, + "AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, "LinearRegression": { "l2": (float, (0.0, 0.0001)), - "optimizer": { - "lr": {"learning_rate": (float, (0.009, 0.011))} - }, - "intercept_lr": {"learning_rate": (float, (0.009, 0.011))} - } + "optimizer": {"lr": {"learning_rate": (float, (0.009, 0.011))}}, + "intercept_lr": {"learning_rate": (float, (0.009, 0.011))}, + }, }, drift_input=lambda yt, yp: abs(yt - yp), drift_detector=drift.PageHinkley(), convergence_sphere=0.000001, - seed=42 + seed=42, ) first_print = True @@ -77,14 +66,14 @@ print(repr(baseline)) print("Best params:") print(repr(sspt.best)) -print("SSPT: ",sspt_metric) -print("Baseline: ",baseline_metric) +print("SSPT: ", sspt_metric) +print("Baseline: ", baseline_metric) -plt.plot(baseline_metric_plt[:10000], linestyle = 'dotted') +plt.plot(baseline_metric_plt[:10000], linestyle="dotted") plt.plot(sspt_metric_plt[:10000]) plt.show() -plt.plot(baseline_rolling_metric_plt[:10000], linestyle = 'dotted') +plt.plot(baseline_rolling_metric_plt[:10000], linestyle="dotted") plt.plot(sspt_rolling_metric_plt[:10000]) -plt.show() \ No newline at end of file +plt.show() diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index a5733b6..9023024 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -1,8 +1,10 @@ import collections import copy import math +import numbers import random import typing + import numpy as np # TODO use lazy imports where needed @@ -68,6 +70,9 @@ def __init__( # Models expanded from the simplex self._expanded: typing.Optional[typing.Dict] = None + # Convergence criterion + self._old_centroid = None + # Meta-programming border = self.estimator if isinstance(border, compose.Pipeline): @@ -80,36 +85,83 @@ def __init__( elif isinstance(border, anomaly.base.AnomalyDetector): self._scorer_name = "classify" - def _random_config(self): - def gen_random(p_data, e_data): - # Sub-component needs to be instantiated - if isinstance(e_data, tuple): - sub_class, sub_data = e_data - sub_config = {} + def __generate(self, p_data) -> numbers.Number: + p_type, p_range = p_data + if p_type == int: + return self._rng.randint(p_range[0], p_range[1]) + elif p_type == float: + return self._rng.uniform(p_range[0], p_range[1]) + + def __combine(self, p_info, param1, param2, func): + + p_type, p_range = p_info + new_val = func(param1, param2) + + # Range sanity checks + if new_val < p_range[0]: + new_val = p_range[0] + if new_val > p_range[1]: + new_val = p_range[1] + + new_val = round(new_val, 0) if p_type == int else new_val + return new_val + + def _recurse_params(self, p_data, e1_data, func=None, *, e2_data=None): + # Sub-component needs to be instantiated + if isinstance(e1_data, tuple): + sub_class, sub_data1 = e1_data + + if e2_data is not None: + _, sub_data2 = e2_data + else: + sub_data2 = None + + sub_config = {} + + for sub_param, sub_info in p_data.items(): + sub_config[sub_param] = self._recurse_params( + sub_info, + sub_data1[sub_param], + func, + e2_data=None if not sub_data2 else sub_data2[sub_param], + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(p_data, tuple): + if func is None: + return self.__generate(p_data) + else: + return self.__combine(p_data, e1_data, e2_data, func) + + # The sub-parameters need to be expanded + config = {} + for p_name, p_info in p_data.items(): + e1_info = e1_data[p_name] - for sub_param, sub_info in p_data.items(): - sub_config[sub_param] = gen_random(sub_info, sub_data[sub_param]) - return sub_class(**sub_config) - - # We reached the numeric parameters - if isinstance(p_data, tuple): - p_type, p_range = p_data - if p_type == int: - return self._rng.randint(p_range[0], p_range[1]) - elif p_type == float: - return self._rng.uniform(p_range[0], p_range[1]) - - # The sub-parameters need to be expanded - config = {} - for p_name, p_info in p_data.items(): - e_info = e_data[p_name] + if e2_data is not None: + e2_info = e2_data[p_name] + else: + e2_info = None + + if not isinstance(p_info, dict): + config[p_name] = self._recurse_params( + p_info, e1_info, func, e2_data=e2_info + ) + else: sub_config = {} for sub_name, sub_info in p_info.items(): - sub_config[sub_name] = gen_random(sub_info, e_info[sub_name]) + sub_config[sub_name] = self._recurse_params( + sub_info, + e1_info[sub_name], + func, + e2_data=None if not e2_info else e2_info[sub_name], + ) config[p_name] = sub_config - return config + return config - return gen_random(self.params_range, self.estimator._get_params()) + def _random_config(self): + return self._recurse_params(self.params_range, self.estimator._get_params()) def _create_simplex(self, model) -> typing.List: # The simplex is divided in: @@ -120,16 +172,16 @@ def _create_simplex(self, model) -> typing.List: simplex[0] = ModelWrapper( self.estimator.clone(self._random_config(), include_attributes=True), - self.metric.clone(include_attributes=True) + self.metric.clone(include_attributes=True), ) simplex[1] = ModelWrapper( model.clone(self._random_config(), include_attributes=True), - self.metric.clone(include_attributes=True) + self.metric.clone(include_attributes=True), ) simplex[2] = ModelWrapper( self.estimator.clone(self._random_config(), include_attributes=True), - self.metric.clone(include_attributes=True) - ) + self.metric.clone(include_attributes=True), + ) return simplex @@ -143,51 +195,14 @@ def _sort_simplex(self): def _gen_new_estimator(self, e1, e2, func): """Generate new configuration given two estimators and a combination function.""" - def apply_operator(param1, param2, p_info, func): - if isinstance(param1, tuple): - sub_class, sub_params1 = param1 - _, sub_params2 = param2 - - sub_config = {} - for sub_param, sub_info in p_info.items(): - sub_config[sub_param] = apply_operator( - sub_params1[sub_param], sub_params2[sub_param], sub_info, func - ) - return sub_class(**sub_config) - if isinstance(p_info, tuple): - p_type, p_range = p_info - new_val = func(param1, param2) - - # Range sanity checks - if new_val < p_range[0]: - new_val = p_range[0] - if new_val > p_range[1]: - new_val = p_range[1] - - new_val = round(new_val, 0) if p_type == int else new_val - return new_val - - # The sub-parameters need to be expanded - config = {} - for p_name, inner_p_info in p_info.items(): - sub_param1 = param1[p_name] - sub_param2 = param2[p_name] + e1_p = e1.estimator._get_params() + e2_p = e2.estimator._get_params() - sub_config = {} - for sub_name, sub_info in inner_p_info.items(): - sub_config[sub_name] = apply_operator( - sub_param1[sub_name], sub_param2[sub_name], sub_info, func - ) - config[p_name] = sub_config - return config - - e1_params = e1.estimator._get_params() - e2_params = e2.estimator._get_params() - - new_config = apply_operator(e1_params, e2_params, self.params_range, func) + new_config = self._recurse_params(self.params_range, e1_p, func, e2_data=e2_p) # Modify the current best contender with the new hyperparameter values new = ModelWrapper( - copy.deepcopy(self._simplex[0].estimator), self.metric.clone(include_attributes=True) + copy.deepcopy(self._simplex[0].estimator), + self.metric.clone(include_attributes=True), ) new.estimator.mutate(new_config) @@ -195,8 +210,6 @@ def apply_operator(param1, param2, p_info, func): def _nelder_mead_expansion(self) -> typing.Dict: """Create expanded models given the simplex models.""" - #print('----------Simplex------------') - #print(self._simplex) expanded = {} # Midpoint between 'best' and 'good' expanded["midpoint"] = self._gen_new_estimator( @@ -223,8 +236,7 @@ def _nelder_mead_expansion(self) -> typing.Dict: expanded["contraction2"] = self._gen_new_estimator( expanded["midpoint"], expanded["reflection"], lambda h1, h2: (h1 + h2) / 2 ) - #print('----------Expanded------------') - #print(expanded) + return expanded def _nelder_mead_operators(self): @@ -307,13 +319,19 @@ def _models_converged(self) -> bool: ] ) - Listv = [list(scaled_params_b.values()),list(scaled_params_g.values()),list(scaled_params_w.values())] + hyper_points = [ + list(scaled_params_b.values()), + list(scaled_params_g.values()), + list(scaled_params_w.values()), + ] - vectors = np.array(Listv) + vectors = np.array(hyper_points) new_centroid = dict(zip(scaled_params_b.keys(), np.mean(vectors, axis=0))) - centroid_distance = utils.math.minkowski_distance(self.old_centroid, new_centroid, p=2) - self.old_centroid = new_centroid - ndim = len(self.params_range) + centroid_distance = utils.math.minkowski_distance( + self._old_centroid, new_centroid, p=2 + ) + self._old_centroid = new_centroid + ndim = len(scaled_params_b) r_sphere = max_dist * math.sqrt((ndim / (2 * (ndim + 1)))) if r_sphere < self.convergence_sphere or centroid_distance == 0: @@ -366,21 +384,39 @@ def _learn_not_converged(self, x, y): scaled_params_g = {} scaled_params_w = {} self._normalize_flattened_hyperspace( - scaled_params_b, self._simplex[0].estimator._get_params(), self.params_range + scaled_params_b, + self._simplex[0].estimator._get_params(), + self.params_range, ) self._normalize_flattened_hyperspace( - scaled_params_g, self._simplex[1].estimator._get_params(), self.params_range + scaled_params_g, + self._simplex[1].estimator._get_params(), + self.params_range, ) self._normalize_flattened_hyperspace( - scaled_params_w, self._simplex[2].estimator._get_params(), self.params_range + scaled_params_w, + self._simplex[2].estimator._get_params(), + self.params_range, + ) + print("----------") + print( + "B:", list(scaled_params_b.values()), "Score:", self._simplex[0].metric + ) + print( + "G:", list(scaled_params_g.values()), "Score:", self._simplex[1].metric + ) + print( + "W:", list(scaled_params_w.values()), "Score:", self._simplex[2].metric + ) + hyper_points = [ + list(scaled_params_b.values()), + list(scaled_params_g.values()), + list(scaled_params_w.values()), + ] + vectors = np.array(hyper_points) + self._old_centroid = dict( + zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) ) - print('----------') - print('B:',list(scaled_params_b.values()),'Score:',self._simplex[0].metric) - print('G:', list(scaled_params_g.values()),'Score:',self._simplex[1].metric) - print('W:', list(scaled_params_w.values()),'Score:',self._simplex[2].metric) - Listv = [list(scaled_params_b.values()), list(scaled_params_g.values()), list(scaled_params_w.values())] - vectors = np.array(Listv) - self.old_centroid = dict(zip(scaled_params_b.keys(), np.mean(vectors, axis=0))) # Update the simplex models using Nelder-Mead heuristics self._nelder_mead_operators() From 81c6861ebbae5d059fa69814e3b4e3e932807b97 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Wed, 7 Sep 2022 11:27:19 +0100 Subject: [PATCH 17/50] create one frankstein method --- river_extra/model_selection/sspt.py | 88 ++++++++++++++--------------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 9023024..bd950d3 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -106,7 +106,14 @@ def __combine(self, p_info, param1, param2, func): new_val = round(new_val, 0) if p_type == int else new_val return new_val - def _recurse_params(self, p_data, e1_data, func=None, *, e2_data=None): + def __flatten(self, prefix, scaled, p_info, e_info): + _, p_range = p_info + interval = p_range[1] - p_range[0] + scaled[prefix] = (e_info - p_range[0]) / interval + + def _recurse_params( + self, p_data, e1_data, func=None, *, e2_data=None, prefix=None, scaled=None + ): # Sub-component needs to be instantiated if isinstance(e1_data, tuple): sub_class, sub_data1 = e1_data @@ -119,20 +126,27 @@ def _recurse_params(self, p_data, e1_data, func=None, *, e2_data=None): sub_config = {} for sub_param, sub_info in p_data.items(): + if prefix is not None: + prefix_ = prefix + "__" + sub_param + sub_config[sub_param] = self._recurse_params( sub_info, sub_data1[sub_param], func, e2_data=None if not sub_data2 else sub_data2[sub_param], + prefix=None if prefix is None else prefix_, + scaled=scaled, ) return sub_class(**sub_config) # We reached the numeric parameters if isinstance(p_data, tuple): - if func is None: + if func is None and prefix is None: return self.__generate(p_data) - else: - return self.__combine(p_data, e1_data, e2_data, func) + if prefix is not None: + self.__flatten(prefix, scaled, p_data, e1_data) + return + return self.__combine(p_data, e1_data, e2_data, func) # The sub-parameters need to be expanded config = {} @@ -144,9 +158,17 @@ def _recurse_params(self, p_data, e1_data, func=None, *, e2_data=None): else: e2_info = None + if prefix is not None: + prefix_ = prefix + "__" + p_name if len(prefix) > 0 else p_name + if not isinstance(p_info, dict): config[p_name] = self._recurse_params( - p_info, e1_info, func, e2_data=e2_info + p_info, + e1_info, + func, + e2_data=e2_info, + prefix=None if prefix is None else prefix_, + scaled=scaled, ) else: sub_config = {} @@ -156,6 +178,8 @@ def _recurse_params(self, p_data, e1_data, func=None, *, e2_data=None): e1_info[sub_name], func, e2_data=None if not e2_info else e2_info[sub_name], + prefix=None if prefix is None else prefix_, + scaled=scaled, ) config[p_name] = sub_config return config @@ -272,43 +296,24 @@ def _nelder_mead_operators(self): self._sort_simplex() - def _normalize_flattened_hyperspace(self, scaled, orig, info, prefix=""): - if isinstance(orig, tuple): - _, sub_orig = orig - for sub_param, sub_info in info.items(): - prefix_ = prefix + "__" + sub_param - self._normalize_flattened_hyperspace( - scaled, sub_orig[sub_param], sub_info, prefix_ - ) - return - - if isinstance(info, tuple): - _, p_range = info - interval = p_range[1] - p_range[0] - scaled[prefix] = (orig - p_range[0]) / interval - return - - for p_name, p_info in info.items(): - sub_orig = orig[p_name] - prefix_ = prefix + "__" + p_name if len(prefix) > 0 else p_name - self._normalize_flattened_hyperspace(scaled, sub_orig, p_info, prefix_) + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._recurse_params(self.params_range, e1_data=orig, prefix="", scaled=scaled) + return scaled @property def _models_converged(self) -> bool: # Normalize params to ensure they contribute equally to the stopping criterion # 1. Simplex in sphere - scaled_params_b = {} - scaled_params_g = {} - scaled_params_w = {} - self._normalize_flattened_hyperspace( - scaled_params_b, self._simplex[0].estimator._get_params(), self.params_range + scaled_params_b = self._normalize_flattened_hyperspace( + self._simplex[0].estimator._get_params() ) - self._normalize_flattened_hyperspace( - scaled_params_g, self._simplex[1].estimator._get_params(), self.params_range + scaled_params_g = self._normalize_flattened_hyperspace( + self._simplex[1].estimator._get_params() ) - self._normalize_flattened_hyperspace( - scaled_params_w, self._simplex[2].estimator._get_params(), self.params_range + scaled_params_w = self._normalize_flattened_hyperspace( + self._simplex[2].estimator._get_params() ) max_dist = max( @@ -380,23 +385,14 @@ def _learn_not_converged(self, x, y): if self._n == self.grace_period: self._n = 0 # 1. Simplex in sphere - scaled_params_b = {} - scaled_params_g = {} - scaled_params_w = {} - self._normalize_flattened_hyperspace( - scaled_params_b, + scaled_params_b = self._normalize_flattened_hyperspace( self._simplex[0].estimator._get_params(), - self.params_range, ) - self._normalize_flattened_hyperspace( - scaled_params_g, + scaled_params_g = self._normalize_flattened_hyperspace( self._simplex[1].estimator._get_params(), - self.params_range, ) - self._normalize_flattened_hyperspace( - scaled_params_w, + scaled_params_w = self._normalize_flattened_hyperspace( self._simplex[2].estimator._get_params(), - self.params_range, ) print("----------") print( From 89ae606f9d6300da417ce157afb730615cc1b523 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Thu, 8 Sep 2022 17:44:35 +0100 Subject: [PATCH 18/50] fix bug in scaling --- river_extra/model_selection/sspt.py | 88 +++++++++++++++++++---------- 1 file changed, 59 insertions(+), 29 deletions(-) diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index bd950d3..b76e15e 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -1,3 +1,4 @@ +from ast import operator import collections import copy import math @@ -112,40 +113,43 @@ def __flatten(self, prefix, scaled, p_info, e_info): scaled[prefix] = (e_info - p_range[0]) / interval def _recurse_params( - self, p_data, e1_data, func=None, *, e2_data=None, prefix=None, scaled=None + self, operation, p_data, e1_data, *, func=None, e2_data=None, prefix=None, scaled=None ): # Sub-component needs to be instantiated if isinstance(e1_data, tuple): sub_class, sub_data1 = e1_data - if e2_data is not None: + if operation == "combine": _, sub_data2 = e2_data else: - sub_data2 = None + sub_data2 = {} sub_config = {} - for sub_param, sub_info in p_data.items(): - if prefix is not None: - prefix_ = prefix + "__" + sub_param + if operation == "scale": + sub_prefix = prefix + "__" + sub_param + else: + sub_prefix = None sub_config[sub_param] = self._recurse_params( - sub_info, - sub_data1[sub_param], - func, - e2_data=None if not sub_data2 else sub_data2[sub_param], - prefix=None if prefix is None else prefix_, + operation=operation, + p_data=sub_info, + e1_data=sub_data1[sub_param], + func=func, + e2_data=sub_data2.get(sub_param, None), + prefix=sub_prefix, scaled=scaled, ) return sub_class(**sub_config) # We reached the numeric parameters if isinstance(p_data, tuple): - if func is None and prefix is None: + if operation == "generate": return self.__generate(p_data) - if prefix is not None: + if operation == "scale": self.__flatten(prefix, scaled, p_data, e1_data) return + # combine return self.__combine(p_data, e1_data, e2_data, func) # The sub-parameters need to be expanded @@ -153,39 +157,53 @@ def _recurse_params( for p_name, p_info in p_data.items(): e1_info = e1_data[p_name] - if e2_data is not None: + if operation == "combine": e2_info = e2_data[p_name] else: - e2_info = None + e2_info = {} - if prefix is not None: - prefix_ = prefix + "__" + p_name if len(prefix) > 0 else p_name + if operation == "scale": + sub_prefix = prefix + "__" + p_name if len(prefix) > 0 else p_name + else: + sub_prefix = None if not isinstance(p_info, dict): config[p_name] = self._recurse_params( - p_info, - e1_info, - func, + operation=operation, + p_data=p_info, + e1_data=e1_info, + func=func, e2_data=e2_info, - prefix=None if prefix is None else prefix_, + prefix=sub_prefix, scaled=scaled, ) else: sub_config = {} for sub_name, sub_info in p_info.items(): + + if operation == "scale": + sub_prefix2 = sub_prefix + "__" + sub_name + else: + sub_prefix2 = None + sub_config[sub_name] = self._recurse_params( - sub_info, - e1_info[sub_name], - func, - e2_data=None if not e2_info else e2_info[sub_name], - prefix=None if prefix is None else prefix_, + operation=operation, + p_data=sub_info, + e1_data=e1_info[sub_name], + func=func, + e2_data=e2_info.get(sub_name, None), + prefix=sub_prefix2, scaled=scaled, ) config[p_name] = sub_config return config def _random_config(self): - return self._recurse_params(self.params_range, self.estimator._get_params()) + return self._recurse_params( + operation="generate", + p_data=self.params_range, + e1_data=self.estimator._get_params() + ) def _create_simplex(self, model) -> typing.List: # The simplex is divided in: @@ -222,7 +240,13 @@ def _gen_new_estimator(self, e1, e2, func): e1_p = e1.estimator._get_params() e2_p = e2.estimator._get_params() - new_config = self._recurse_params(self.params_range, e1_p, func, e2_data=e2_p) + new_config = self._recurse_params( + operation="combine", + p_data=self.params_range, + e1_data=e1_p, + func=func, + e2_data=e2_p + ) # Modify the current best contender with the new hyperparameter values new = ModelWrapper( copy.deepcopy(self._simplex[0].estimator), @@ -298,7 +322,13 @@ def _nelder_mead_operators(self): def _normalize_flattened_hyperspace(self, orig): scaled = {} - self._recurse_params(self.params_range, e1_data=orig, prefix="", scaled=scaled) + self._recurse_params( + operation="scale", + p_data=self.params_range, + e1_data=orig, + prefix="", + scaled=scaled + ) return scaled @property From ec221810c63b1b7a1231141c215d1cef8bed0b3d Mon Sep 17 00:00:00 2001 From: Max Halford Date: Wed, 24 Aug 2022 11:00:51 +0200 Subject: [PATCH 19/50] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4999689..de50559 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This package contains additional estimators that have not been put into the main ## Installation ```sh -pip install river[extra] +pip install "river[extra]" ``` You can also install from source: From ccfc25ed003e42f14fb4f27827ca095055d139ae Mon Sep 17 00:00:00 2001 From: Max Halford Date: Wed, 24 Aug 2022 11:12:56 +0200 Subject: [PATCH 20/50] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 774cef0..24a8527 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "BSD-3" [tool.poetry.dependencies] python = ">=3.6.1,<4.0" -river = "*" +river = "^0.11.1" [tool.poetry.dev-dependencies] From 6c55a140c81b81a294b6accf9f836baf19f918f8 Mon Sep 17 00:00:00 2001 From: Max Halford Date: Wed, 24 Aug 2022 11:13:13 +0200 Subject: [PATCH 21/50] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 24a8527..ea37971 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "BSD-3" [tool.poetry.dependencies] python = ">=3.6.1,<4.0" -river = "^0.11.1" +river = "^0.10" [tool.poetry.dev-dependencies] From a7e63d5cc413a36ec131cb6c1b721cd937a9fc7d Mon Sep 17 00:00:00 2001 From: Max Halford Date: Wed, 24 Aug 2022 11:24:48 +0200 Subject: [PATCH 22/50] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ea37971..29df41a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "river_extra" -version = "0.8.0" +version = "0.11.1" description = "Additional estimators for the River package" authors = ["MaxHalford "] license = "BSD-3" From aea5fc3ef641afb6e8b19d99e37960fbd5cc7921 Mon Sep 17 00:00:00 2001 From: Max Halford Date: Wed, 24 Aug 2022 12:10:20 +0200 Subject: [PATCH 23/50] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 29df41a..3c6fa94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "BSD-3" [tool.poetry.dependencies] python = ">=3.6.1,<4.0" -river = "^0.10" +river = "0.11.1" [tool.poetry.dev-dependencies] From 02f5cb2f7bb656204945ca8469a6b25dc235270c Mon Sep 17 00:00:00 2001 From: Max Halford Date: Wed, 24 Aug 2022 14:39:47 +0200 Subject: [PATCH 24/50] fix river imports --- .gitignore | 1 + river_extra/metrics/__init__.py | 14 ++++++++++++++ river_extra/metrics/cluster/__init__.py | 1 - river_extra/metrics/kappa.py | 4 +++- river_extra/metrics/prevalence_threshold.py | 2 +- river_extra/metrics/purity.py | 2 +- river_extra/metrics/q0.py | 4 ++-- river_extra/metrics/variation_info.py | 4 ++-- 8 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 river_extra/metrics/__init__.py diff --git a/.gitignore b/.gitignore index 44eccf2..f9446a5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ poetry.lock *.whl *.tar.gz *.pyc +/*.ipynb diff --git a/river_extra/metrics/__init__.py b/river_extra/metrics/__init__.py new file mode 100644 index 0000000..508c834 --- /dev/null +++ b/river_extra/metrics/__init__.py @@ -0,0 +1,14 @@ +from . import cluster +from .purity import Purity +from .prevalence_threshold import PrevalenceThreshold +from .q0 import Q0, Q2 +from .variation_info import VariationInfo + +__all__ = [ + "cluster", + "Purity", + "PrevalenceThreshold", + "Q0", + "Q2", + "VariationInfo" +] diff --git a/river_extra/metrics/cluster/__init__.py b/river_extra/metrics/cluster/__init__.py index af09afc..491686f 100644 --- a/river_extra/metrics/cluster/__init__.py +++ b/river_extra/metrics/cluster/__init__.py @@ -15,7 +15,6 @@ from .rmsstd import MSSTD, RMSSTD from .sd_validation import SD from .separation import Separation -from .silhouette import Silhouette from .ssb import SSB from .ssq_based import WB, CalinskiHarabasz, Hartigan from .ssw import SSW, BallHall, Cohesion, Xu diff --git a/river_extra/metrics/kappa.py b/river_extra/metrics/kappa.py index 26665b2..cc7bf66 100644 --- a/river_extra/metrics/kappa.py +++ b/river_extra/metrics/kappa.py @@ -1,4 +1,6 @@ -class KappaM(base.MultiClassMetric): +from river import metrics + +class KappaM(metrics.base.MultiClassMetric): r"""Kappa-M score. The Kappa-M statistic compares performance with the majority class classifier. diff --git a/river_extra/metrics/prevalence_threshold.py b/river_extra/metrics/prevalence_threshold.py index 0e0a8cf..0fe2acd 100644 --- a/river_extra/metrics/prevalence_threshold.py +++ b/river_extra/metrics/prevalence_threshold.py @@ -5,7 +5,7 @@ __all__ = ["PrevalenceThreshold"] -class PrevalenceThreshold(metrics.BinaryMetric): +class PrevalenceThreshold(metrics.base.BinaryMetric): r"""Prevalence Threshold (PT). The relationship between a positive predicted value and its target prevalence diff --git a/river_extra/metrics/purity.py b/river_extra/metrics/purity.py index 082ff16..6794bad 100644 --- a/river_extra/metrics/purity.py +++ b/river_extra/metrics/purity.py @@ -3,7 +3,7 @@ __all__ = ["Purity"] -class Purity(metrics.MultiClassMetric): +class Purity(metrics.base.MultiClassMetric): r"""Purity. In a similar fashion with Entropy, the purity of a clustering solution, diff --git a/river_extra/metrics/q0.py b/river_extra/metrics/q0.py index b01ef23..c10cc73 100644 --- a/river_extra/metrics/q0.py +++ b/river_extra/metrics/q0.py @@ -2,12 +2,12 @@ from scipy.special import factorial -from . import base +from river import metrics __all__ = ["Q0", "Q2"] -class Q0(base.MultiClassMetric): +class Q0(metrics.base.MultiClassMetric): r"""Q0 index. Dom's Q0 measure [^2] uses conditional entropy to calculate the goodness of diff --git a/river_extra/metrics/variation_info.py b/river_extra/metrics/variation_info.py index 4440279..867a83a 100644 --- a/river_extra/metrics/variation_info.py +++ b/river_extra/metrics/variation_info.py @@ -1,11 +1,11 @@ import math -from . import base +from river import metrics __all__ = ["VariationInfo"] -class VariationInfo(base.MultiClassMetric): +class VariationInfo(metrics.base.MultiClassMetric): r"""Variation of Information. Variation of Information (VI) [^1] [^2] is an information-based clustering measure. From dc65a4b88412e769e88ccb79bf2ebe2fcc6e9171 Mon Sep 17 00:00:00 2001 From: Max Halford Date: Wed, 24 Aug 2022 15:04:11 +0200 Subject: [PATCH 25/50] fix clustering imports --- river_extra/metrics/cluster/bic.py | 3 ++- river_extra/metrics/cluster/ssq_based.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/river_extra/metrics/cluster/bic.py b/river_extra/metrics/cluster/bic.py index 6795d80..824d725 100644 --- a/river_extra/metrics/cluster/bic.py +++ b/river_extra/metrics/cluster/bic.py @@ -3,6 +3,7 @@ from river import metrics from . import base +from .ssw import SSW class BIC(base.ClusteringMetric): @@ -84,7 +85,7 @@ class BIC(base.ClusteringMetric): def __init__(self): super().__init__() - self._ssw = metrics.cluster.SSW() + self._ssw = SSW() self._n_points_by_clusters = {} self._n_clusters = 0 self._dim = 0 diff --git a/river_extra/metrics/cluster/ssq_based.py b/river_extra/metrics/cluster/ssq_based.py index a74292f..c8f85ee 100644 --- a/river_extra/metrics/cluster/ssq_based.py +++ b/river_extra/metrics/cluster/ssq_based.py @@ -3,6 +3,8 @@ from river import metrics from . import base +from .ssb import SSB +from .ssw import SSW __all__ = ["CalinskiHarabasz", "Hartigan", "WB"] @@ -57,8 +59,8 @@ class CalinskiHarabasz(base.ClusteringMetric): def __init__(self): super().__init__() - self._ssb = metrics.cluster.SSB() - self._ssw = metrics.cluster.SSW() + self._ssb = SSB() + self._ssw = SSW() self._n_clusters = 0 self._n_points = 0 From 50cde88390d74573a0d9fe91ec715b2f5be2dd74 Mon Sep 17 00:00:00 2001 From: Max Halford Date: Fri, 2 Sep 2022 21:32:01 +0200 Subject: [PATCH 26/50] Update pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3c6fa94..e9a99cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] name = "river_extra" -version = "0.11.1" +version = "0.12.1" description = "Additional estimators for the River package" authors = ["MaxHalford "] license = "BSD-3" [tool.poetry.dependencies] python = ">=3.6.1,<4.0" -river = "0.11.1" +river = "0.12.1" [tool.poetry.dev-dependencies] From 76eb29495e962f0989f1b8fd7c443967851f01ce Mon Sep 17 00:00:00 2001 From: Max Halford Date: Sat, 17 Sep 2022 14:42:11 +0200 Subject: [PATCH 27/50] 0.13.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e9a99cb..4f3deb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "river_extra" -version = "0.12.1" +version = "0.13.0" description = "Additional estimators for the River package" authors = ["MaxHalford "] license = "BSD-3" From e93e840fcea372ce71edc80f17c88ec60f00c3d4 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Fri, 30 Sep 2022 16:47:59 -0300 Subject: [PATCH 28/50] add OXT --- .gitignore | 1 + pyproject.toml | 2 +- river_extra/__version__.py | 2 +- river_extra/ensemble/__init__.py | 3 + river_extra/ensemble/online_extra_trees.py | 726 +++++++++++++++++++ river_extra/tree/nodes/__init__.py | 0 river_extra/tree/nodes/et_nodes.py | 68 ++ river_extra/tree/splitter/__init__.py | 3 + river_extra/tree/splitter/random_splitter.py | 93 +++ 9 files changed, 896 insertions(+), 2 deletions(-) create mode 100644 river_extra/ensemble/__init__.py create mode 100644 river_extra/ensemble/online_extra_trees.py create mode 100644 river_extra/tree/nodes/__init__.py create mode 100644 river_extra/tree/nodes/et_nodes.py create mode 100644 river_extra/tree/splitter/__init__.py create mode 100644 river_extra/tree/splitter/random_splitter.py diff --git a/.gitignore b/.gitignore index f9446a5..b65cd85 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ poetry.lock *.tar.gz *.pyc /*.ipynb +.DS_Store diff --git a/pyproject.toml b/pyproject.toml index 4f3deb7..c1db5b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "BSD-3" [tool.poetry.dependencies] python = ">=3.6.1,<4.0" -river = "0.12.1" +river = "0.13.0" [tool.poetry.dev-dependencies] diff --git a/river_extra/__version__.py b/river_extra/__version__.py index 0deec50..e5ecb06 100644 --- a/river_extra/__version__.py +++ b/river_extra/__version__.py @@ -1,3 +1,3 @@ -VERSION = (0, 1, 0) +VERSION = (0, 13, 0) __version__ = ".".join(map(str, VERSION)) diff --git a/river_extra/ensemble/__init__.py b/river_extra/ensemble/__init__.py new file mode 100644 index 0000000..0820ac9 --- /dev/null +++ b/river_extra/ensemble/__init__.py @@ -0,0 +1,3 @@ +from .online_extra_trees import ExtraTreesRegressor + +__all__ = ["ExtraTreesRegressor"] diff --git a/river_extra/ensemble/online_extra_trees.py b/river_extra/ensemble/online_extra_trees.py new file mode 100644 index 0000000..e6f9991 --- /dev/null +++ b/river_extra/ensemble/online_extra_trees.py @@ -0,0 +1,726 @@ +import abc +import collections +import copy +import math +import random +import sys +import typing + +from river import base, drift, metrics, tree + +from ..tree.nodes.et_nodes import ETLeafAdaptive, ETLeafMean, ETLeafModel +from ..tree.splitter import RegRandomSplitter + + +class ExtraTrees(base.Ensemble, metaclass=abc.ABCMeta): + _FEATURES_SQRT = "sqrt" + _FEATURES_LOG2 = "log2" + _FEATURES_RANDOM = "random" + + _BAGGING = "bagging" + _SUBBAGGING = "subbagging" + + _DETECTION_ALL = "all" + _DETECTION_DROP = "drop" + _DETECTION_OFF = "off" + + def __init__( + self, + n_models: int, + max_features: typing.Union[bool, str, int], + resampling_strategy: typing.Optional[str], + resampling_rate: typing.Union[int, float], + detection_mode: str, + warning_detector: base.DriftDetector, + drift_detector: base.DriftDetector, + max_depth: typing.Union[str, int], + randomize_tree_depth: bool, + track_metric: typing.Union[metrics.base.MultiClassMetric, metrics.base.RegressionMetric], + disable_weighted_vote: bool, + split_buffer_size: int, + seed: int, + ): + self.data = [] + self.n_models = n_models + self.max_features = max_features + + if resampling_strategy not in [None, self._BAGGING, self._SUBBAGGING]: + raise ValueError(f"Invalid resampling strategy: {resampling_strategy}") + self.resampling_strategy = resampling_strategy + + if self.resampling_strategy is not None: + if self.resampling_strategy == self._BAGGING: + if resampling_rate < 1: + raise ValueError( + "'resampling_rate' must be an integer greater than or" + "equal to 1, when resample_strategy='bagging'." + ) + # Cast to integer (online bagging using poisson sampling) + self.resampling_rate = int(resampling_rate) + + if self.resampling_strategy == self._SUBBAGGING: + if not 0 < resampling_rate <= 1: + raise ValueError( + "resampling_rate must be a float in the interval (0, 1]," + "when resampling_strategy='subbagging'." + ) + self.resampling_rate = resampling_rate + + if detection_mode not in [ + self._DETECTION_ALL, + self._DETECTION_DROP, + self._DETECTION_OFF, + ]: + raise ValueError( + f"Invalid drift detection mode. Valid values are: '{self._DETECTION_ALL}'," + f" {self._DETECTION_DROP}, and '{self._DETECTION_OFF}'." + ) + + self.detection_mode = detection_mode + self.warning_detector = ( + warning_detector if warning_detector is not None else drift.ADWIN(delta=0.01) + ) + self.drift_detector = ( + drift_detector if drift_detector is not None else drift.ADWIN(delta=0.001) + ) + + self.max_depth = max_depth + self.randomize_tree_depth = randomize_tree_depth + self.track_metric = track_metric + self.disable_weighted_vote = disable_weighted_vote + self.split_buffer_size = split_buffer_size + self.seed = seed + + # The predictive performance of each tree + self._perfs: typing.List = [] + # Keep a running estimate of the sum of performances + self._perf_sum: float = 0 + + # Number of times a tree will use each instance to learn from it + self._weight_sampler = self.__weight_sampler_factory() + + # General statistics + # Counter of the number of instances each ensemble member has processed (instance weights + # are not accounted for, just the number of instances) + self._sample_counter = collections.Counter() + # Total of samples processed by the Extra Trees ensemble + self._total_instances: float = 0 + # Number of warnings triggered + self._n_warnings = collections.Counter() + # Number of drifts detected + self._n_drifts = collections.Counter() + # Number of tree swaps + self._n_tree_swaps = collections.Counter() + + self._background_trees = {} + # Initialize drift detectors and select the detection mode procedure + if self.detection_mode == self._DETECTION_ALL: + self._warn_detectors = {i: self.warning_detector.clone() for i in range(self.n_models)} + self._drift_detectors = {i: self.drift_detector.clone() for i in range(self.n_models)} + elif self.detection_mode == self._DETECTION_DROP: + self._warn_detectors = {} + self._drift_detectors = {i: self.drift_detector.clone() for i in range(self.n_models)} + else: # detection_mode: "off" + self._warn_detectors = {} + self._drift_detectors = {} + self._detect = self.__detection_mode_factory() + + # Set the rng + self._rng = random.Random(seed) + + @abc.abstractmethod + def _new_member( + self, max_features, max_depth, seed + ) -> typing.Union[base.Classifier, base.Regressor]: + pass + + @abc.abstractmethod + def _drift_input(self, y, y_hat) -> typing.Union[int, float]: + pass + + def _calculate_tree_depth(self) -> float: + if self.max_depth is None: + return math.inf + + if not self.randomize_tree_depth: + return self.max_depth + else: # Randomize tree depth + return self._rng.randint(1, self.max_depth if not math.isinf(self.max_depth) else 9999) + + def _calculate_max_features(self, n_features) -> int: + if self.max_features == self._FEATURES_RANDOM: + # Generate a random integer + return self._rng.randint(2, n_features) + else: + if self.max_features == self._FEATURES_SQRT: + max_feat = round(math.sqrt(n_features)) + elif self.max_features == self._FEATURES_LOG2: + max_feat = round(math.log2(n_features)) + elif isinstance(self.max_features, int): + max_feat = n_features + elif isinstance(self.max_features, float): + # Consider 'max_features' as a percentage + max_feat = int(self.max_features * n_features) + elif self.max_features is None: + max_feat = n_features + else: + raise AttributeError( + f"Invalid max_features: {self.max_features}.\n" + f"Valid options are: int [2, M], float (0., 1.]," + f" {self._FEATURES_SQRT}, {self._FEATURES_LOG2}" + ) + + # Sanity checks + # max_feat is negative, use max_feat + n + if max_feat < 0: + max_feat += n_features + # max_feat <= 0 + # (m can be negative if max_feat is negative and abs(max_feat) > n), + # use max_features = 1 + if max_feat <= 0: + max_feat = 1 + # max_feat > n, then use n + if max_feat > n_features: + max_feat = n_features + + return max_feat + + def _init_trees(self, n_features: int): + for i in range(self.n_models): + self.data.append( + self._new_member( + max_features=self._calculate_max_features(n_features), + max_depth=self._calculate_tree_depth(), + seed=self._rng.randint(0, sys.maxsize), # randomly creates a new seed + ) + ) + self._perfs.append(copy.deepcopy(self.track_metric)) + + # TODO check if it can be pickled + def __weight_sampler_factory(self): + def constant_sampler(): + return 1 + + def bagging_sampler(): + return self._poisson_sample(self.resampling_rate) + + def subbagging_sampler(): + return 1 if self._rng.random() <= self.resampling_rate else 0 + + if self.resampling_strategy == self._BAGGING: + return bagging_sampler + elif self.resampling_strategy == self._SUBBAGGING: + return subbagging_sampler + else: + return constant_sampler + + # TODO check if there is a more elegant solution + def __detection_mode_factory(self): + def detection_mode_all(drift_detector, warning_detector, detector_input): + in_warning = warning_detector.update(detector_input).drift_detected + in_drift = drift_detector.update(detector_input).drift_detected + + return in_drift, in_warning + + def detection_mode_drop(drift_detector, warning_detector, detector_input): + in_drift = drift_detector.update(detector_input).drift_detected + + return in_drift, False + + def detection_mode_off(drift_detector, warning_detector, detector_input): + return False, False + + if self.detection_mode == self._DETECTION_ALL: + return detection_mode_all + elif self.detection_mode == self._DETECTION_DROP: + return detection_mode_drop + else: + return detection_mode_off + + def learn_one(self, x, y): + if not self.models: + self._init_trees(len(x)) + + self._total_instances += 1 + trained = [] + for i, model in enumerate(self.models): + y_hat = model.predict_one(x) + in_drift, in_warning = self._detect( + self._drift_detectors.get(i), + self._warn_detectors.get(i), + self._drift_input(y, y_hat), + ) + + if in_warning: + self._background_trees[i] = self._new_member( + max_features=self._calculate_max_features(len(x)), + max_depth=self._calculate_tree_depth(), + seed=self._rng.randint(0, sys.maxsize), # randomly creates a new seed + ) + # Reset the warning detector + self._warn_detectors[i] = self.warning_detector.clone() + # Update statistics + self._n_warnings.update([i]) + + # Drift detected: time to change (swap or reset) the affected tree + if in_drift: + if i in self._background_trees: + self.data[i] = self._background_trees[i] + del self._background_trees[i] + + self._n_tree_swaps.update([i]) + else: + self.data[i] = self._new_member( + max_features=self._calculate_max_features(len(x)), + max_depth=self._calculate_tree_depth(), + seed=self._rng.randint(0, sys.maxsize), # randomly creates a new seed + ) + # Reset the drift detector + self._drift_detectors[i] = self.drift_detector.clone() + # Update statistics + self._n_drifts.update([i]) + # Also reset tree's error estimates + self._perf_sum -= self._perfs[i].get() + self._perfs[i] = copy.deepcopy(self.track_metric) + self._perf_sum += self._perfs[i].get() + # And the number of observations of the new model + self._sample_counter[i] = 0 + + # Remove the old performance estimate + self._perf_sum -= self._perfs[i].get() + # Update metric + self._perfs[i].update(y, y_hat) + # Add the new performance estimate + self._perf_sum += self._perfs[i].get() + + # Define the weight of the instance + w = self._weight_sampler() + if w == 0: # Skip model update if w is zero + continue + + model.learn_one(x, y, sample_weight=w) + + if i in self._background_trees: + self._background_trees[i].learn_one(x, y, sample_weight=w) + + trained.append(i) + + # Increase by one the count of instances observed by each trained model + self._sample_counter.update(trained) + + return self + + def _poisson_sample(self, lambda_value) -> int: + """Helper function to sample from poisson distributions without relying on numpy.""" + l_val = math.exp(-lambda_value) + k = 0 + p = 1 + + while p > l_val: + k += 1 + p *= self._rng.random() + + return k - 1 + + # Properties + @property + def n_warnings(self) -> collections.Counter: + """The number of warnings detected per ensemble member.""" + return self._n_warnings + + @property + def n_drifts(self) -> collections.Counter: + """The number of concept drifts detected per ensemble member.""" + return self._n_drifts + + @property + def n_tree_swaps(self) -> collections.Counter: + """The number of performed alternate tree swaps. + + Not applicable if the warning detectors are disabled. + """ + return self._n_tree_swaps + + @property + def total_instances(self) -> float: + """The total number of instances processed by the ensemble.""" + return self._total_instances + + @property + def instances_per_tree(self) -> collections.Counter: + """The number of instances processed by each one of the current forest members. + + Each time a concept drift is detected, the count corresponding to the affected tree is + reset. + """ + return self._sample_counter + + +class ETRegressor(tree.HoeffdingTreeRegressor): + """Extra Tree regressor. + + This is the base-estimator of the Extra Trees regressor. + This variant of the Hoeffding Tree regressor includes the `max_features` parameter, + which defines the number of randomly selected features to be considered at each split. + It also evaluates split candidates randomly. + + """ + + def __init__( + self, + max_features, + grace_period, + max_depth, + delta, + tau, + leaf_prediction, + leaf_model, + model_selector_decay, + nominal_attributes, + min_samples_split, + binary_split, + max_size, + memory_estimate_period, + stop_mem_management, + remove_poor_attrs, + merit_preprune, + split_buffer_size, + seed, + ): + self.max_features = max_features + self.split_buffer_size = split_buffer_size + self.seed = seed + self._rng = random.Random(self.seed) + + super().__init__( + grace_period=grace_period, + max_depth=max_depth, + delta=delta, + tau=tau, + leaf_prediction=leaf_prediction, + leaf_model=leaf_model, + model_selector_decay=model_selector_decay, + nominal_attributes=nominal_attributes, + splitter=RegRandomSplitter( + seed=self._rng.randint(0, sys.maxsize), + buffer_size=self.split_buffer_size, + ), + min_samples_split=min_samples_split, + binary_split=binary_split, + max_size=max_size, + memory_estimate_period=memory_estimate_period, + stop_mem_management=stop_mem_management, + remove_poor_attrs=remove_poor_attrs, + merit_preprune=merit_preprune, + ) + + def _new_learning_node(self, initial_stats=None, parent=None): # noqa + """Create a new learning node. + The type of learning node depends on the tree configuration. + """ + + if parent is not None: + depth = parent.depth + 1 + else: + depth = 0 + + # Generate a random seed for the new learning node + seed = self._rng.randint(0, sys.maxsize) + + leaf_model = None + if self.leaf_prediction in {self._MODEL, self._ADAPTIVE}: + if parent is None: + leaf_model = copy.deepcopy(self.leaf_model) + else: + leaf_model = copy.deepcopy(parent._leaf_model) # noqa + + if self.leaf_prediction == self._TARGET_MEAN: + return ETLeafMean( + initial_stats, + depth, + self.splitter, + self.max_features, + seed, + ) + elif self.leaf_prediction == self._MODEL: + return ETLeafModel( + initial_stats, + depth, + self.splitter, + self.max_features, + seed, + leaf_model=leaf_model, + ) + else: # adaptive learning node + new_adaptive = ETLeafAdaptive( + initial_stats, + depth, + self.splitter, + self.max_features, + seed, + leaf_model=leaf_model, + ) + if parent is not None and isinstance(parent, ETLeafAdaptive): + new_adaptive._fmse_mean = parent._fmse_mean # noqa + new_adaptive._fmse_model = parent._fmse_model # noqa + + return new_adaptive + + +class ExtraTreesRegressor(ExtraTrees, base.Regressor): + """Online Extra Trees regressor. + + The online Extra Trees ensemble takes some steps further into randomization when + compared to Adaptive Random Forests (ARF). A subspace of the feature space is considered + at each split attempt, as ARF does, and online bagging or subbagging can also be + (optionally) used. Nonetheless, Extra Trees randomizes the split candidates evaluated by each + leaf node (just a single split is tested by numerical feature, which brings significant + speedups to the ensemble), and might also randomize the maximum depth of the forest members, + as well as the size of the feature subspace processed by each of its trees' leaves. + + Parameters + ---------- + n_models + The number of trees in the ensemble. + max_features + Max number of attributes for each node split.
+ - If int, then consider `max_features` at each split.
+ - If float, then `max_features` is a percentage and `int(max_features * n_features)` + features are considered per split.
+ - If "sqrt", then `max_features=sqrt(n_features)`.
+ - If "log2", then `max_features=log2(n_features)`.
+ - If "random", then `max_features` will assume a different random number in the interval + `[2, n_features]` for each tree leaf.
+ - If None, then `max_features=n_features`. + resampling_strategy + The chosen instance resampling strategy:
+ - If `None`, no resampling will be done and the trees will process all instances. + - If `'baggging'`, online bagging will be performed (sampling with replacement). + - If `'subbagging'`, online subbagging will be performed (sampling without replacement). + resampling_rate + Only valid if `resampling_strategy` is not None. Controls the parameters of the resampling + strategy.
. + - If `resampling_strategy='bagging'`, must be an integer greater than or equal to 1 that + parameterizes the poisson distribution used to simulate bagging in online learning + settings. It acts as the lambda parameter of Oza Bagging and Leveraging Bagging.
+ - If `resampling_strategy='subbagging'`, must be a float in the interval $(0, 1]$ that + controls the chance of each instance being used by a tree for learning. + detection_mode + The concept drift detection mode in which the forest operates. Valid values are:
+ - "all": creates both warning and concept drift detectors. If a warning is detected, + an alternate tree starts being trained in the background. If the warning trigger escalates + to a concept drift, the affected tree is replaced by the alternate tree.
+ - "drop": only the concept drift detectors are created. If a drift is detected, the + affected tree is dropped and replaced by a new tree.
+ - "off": disables the concept drift adaptation capabilities. The forest will act as if + the processed stream is stationary. + warning_detector + The detector that will be used to trigger concept drift warnings. + drift_detector + The detector used to detect concept drifts. + max_depth + The maximum depth the ensemble members might reach. If `None`, the trees will grow + indefinitely. + randomize_tree_depth + Whether or not randomize the maximum depth of each tree in the ensemble. If `max_depth` + is provided, it is going to act as an upper bound to generate the maximum depth for each + tree. + track_metric + The performance metric used to weight predictions. + disable_weighted_vote + Defines whether or not to use predictions weighted by each trees' prediction performance. + split_buffer_size + Defines the size of the buffer used by the tree splitters when determining the feature + range and a random split point in this interval. + seed + Random seed to support reproducibility. + grace_period + [*Tree parameter*] Number of instances a leaf should observe between + split attempts. + max_depth + [*Tree parameter*] The maximum depth a tree can reach. If `None`, the + tree will grow indefinitely. + delta + [*Tree parameter*] Allowed error in split decision, a value closer to 0 + takes longer to decide. + tau + [*Tree parameter*] Threshold below which a split will be forced to break + ties. + leaf_prediction + [*Tree parameter*] Prediction mechanism used at leaves.
+ - 'mean' - Target mean
+ - 'model' - Uses the model defined in `leaf_model`
+ - 'adaptive' - Chooses between 'mean' and 'model' dynamically
+ leaf_model + [*Tree parameter*] The regression model used to provide responses if + `leaf_prediction='model'`. If not provided, an instance of + `river.linear_model.LinearRegression` with the default hyperparameters + is used. + model_selector_decay + [*Tree parameter*] The exponential decaying factor applied to the learning models' + squared errors, that are monitored if `leaf_prediction='adaptive'`. Must be + between `0` and `1`. The closer to `1`, the more importance is going to + be given to past observations. On the other hand, if its value + approaches `0`, the recent observed errors are going to have more + influence on the final decision. + nominal_attributes + [*Tree parameter*] List of Nominal attributes. If empty, then assume that + all attributes are numerical. + min_samples_split + [*Tree parameter*] The minimum number of samples every branch resulting from a split + candidate must have to be considered valid. + binary_split + [*Tree parameter*] If True, only allow binary splits. + max_size + [*Tree parameter*] Maximum memory (MB) consumed by the tree. + memory_estimate_period + [*Tree parameter*] Number of instances between memory consumption checks. + stop_mem_management + [*Tree parameter*] If True, stop growing as soon as memory limit is hit. + remove_poor_attrs + [*Tree parameter*] If True, disable poor attributes to reduce memory usage. + merit_preprune + [*Tree parameter*] If True, enable merit-based tree pre-pruning. + + Notes + ----- + As the Online Extra Trees change the way in which Hoeffding Trees perform split attempts + and monitor numerical input features, some of the parameters of the vanilla Hoeffding Tree + algorithms are not available. + + Examples + -------- + + >>> from river import datasets + >>> from river import evaluate + >>> from river import metrics + + >>> from river_extra import ensemble + + >>> dataset = dataset.synth.Friedman(seed=42).take(2000) + + >>> model = ensemble.ExtraTreesRegressor( + ... n_models=3, + ... seed=42 + ... ) + + >>> metric = metrics.RMSE() + + >>> evaluate.progressive_val_score(dataset, model, metric) + RMSE: 3.24238 + + """ + + def __init__( + self, + n_models: int = 10, + max_features: typing.Union[bool, str, int] = "random", + resampling_strategy: typing.Optional[str] = None, + resampling_rate: typing.Union[int, float] = 0.5, + detection_mode: str = "all", + warning_detector: base.DriftDetector = None, + drift_detector: base.DriftDetector = None, + max_depth: typing.Union[str, int] = None, + randomize_tree_depth: bool = False, + track_metric: metrics.base.RegressionMetric = metrics.MAE(), + disable_weighted_vote: bool = True, + split_buffer_size: int = 5, + seed: int = None, + grace_period: int = 50, + delta: float = 0.01, + tau: float = 0.05, + leaf_prediction: str = "model", + leaf_model: base.Regressor = None, + model_selector_decay: float = 0.95, + nominal_attributes: list = None, + min_samples_split: int = 5, + binary_split: bool = False, + max_size: int = 500, + memory_estimate_period: int = 2_000_000, + stop_mem_management: bool = False, + remove_poor_attrs: bool = False, + merit_preprune: bool = True, + ): + super().__init__( + n_models=n_models, + max_features=max_features, + resampling_strategy=resampling_strategy, + resampling_rate=resampling_rate, + detection_mode=detection_mode, + warning_detector=warning_detector, + drift_detector=drift_detector, + max_depth=max_depth, + randomize_tree_depth=randomize_tree_depth, + track_metric=track_metric, + disable_weighted_vote=disable_weighted_vote, + split_buffer_size=split_buffer_size, + seed=seed, + ) + + # Tree parameters + self.grace_period = grace_period + self.delta = delta + self.tau = tau + self.leaf_prediction = leaf_prediction + self.leaf_model = leaf_model + self.model_selector_decay = model_selector_decay + self.nominal_attributes = nominal_attributes + self.min_samples_split = min_samples_split + self.binary_split = binary_split + self.max_size = max_size + self.memory_estimate_period = memory_estimate_period + self.stop_mem_management = stop_mem_management + self.remove_poor_attrs = remove_poor_attrs + self.merit_preprune = merit_preprune + + def _new_member(self, max_features, max_depth, seed): + return ETRegressor( + max_features=max_features, + grace_period=self.grace_period, + max_depth=max_depth, + delta=self.delta, + tau=self.tau, + leaf_prediction=self.leaf_prediction, + leaf_model=self.leaf_model, + model_selector_decay=self.model_selector_decay, + nominal_attributes=self.nominal_attributes, + min_samples_split=self.min_samples_split, + binary_split=self.binary_split, + max_size=self.max_size, + memory_estimate_period=self.memory_estimate_period, + stop_mem_management=self.stop_mem_management, + remove_poor_attrs=self.remove_poor_attrs, + merit_preprune=self.merit_preprune, + split_buffer_size=self.split_buffer_size, + seed=seed, + ) + + def _drift_input(self, y, y_hat) -> typing.Union[int, float]: + return abs(y - y_hat) + + def predict_one(self, x: dict) -> base.typing.RegTarget: + if not self.models: + self._init_trees(len(x)) + return 0 + + if not self.disable_weighted_vote: + preds = [] + weights = [] + + for perf, model in zip(self._perfs, self.models): + preds.append(model.predict_one(x)) + weights.append(perf.get()) + + sum_weights = sum(weights) + if sum_weights != 0: + if self.track_metric.bigger_is_better: + preds = [(w / sum_weights) * pred for w, pred in zip(weights, preds)] + else: + weights = [(1 + 1e-8) / (w + 1e-8) for w in weights] + sum_weights = sum(weights) + preds = [(w / sum_weights) * pred for w, pred in zip(weights, preds)] + return sum(preds) + else: + preds = [model.predict_one(x) for model in self.models] + + return sum(preds) / len(preds) diff --git a/river_extra/tree/nodes/__init__.py b/river_extra/tree/nodes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/river_extra/tree/nodes/et_nodes.py b/river_extra/tree/nodes/et_nodes.py new file mode 100644 index 0000000..dfaa0bb --- /dev/null +++ b/river_extra/tree/nodes/et_nodes.py @@ -0,0 +1,68 @@ +import random +import typing + +from river.tree.nodes.htc_nodes import LeafMajorityClass +from river.tree.nodes.htr_nodes import LeafAdaptive, LeafMean, LeafModel +from river.tree.nodes.leaf import HTLeaf + + +class ETLeaf(HTLeaf): + """The Extra Tree leaves change the way in which the splitters are updated + (by using subsets of features). + + Parameters + ---------- + stats + Initial class observations. + depth + The depth of the node. + splitter + The numeric attribute observer algorithm used to monitor target statistics + and perform split attempts. + max_features + Number of attributes per subset for each node split. + seed + Seed to ensure reproducibility. + kwargs + Other parameters passed to the learning node. + """ + + def __init__(self, stats, depth, splitter, max_features, seed, **kwargs): + super().__init__(stats, depth, splitter, **kwargs) + self.max_features = max_features + self.seed = seed + self._rng = random.Random(self.seed) + self.feature_indices = [] + + def _iter_features(self, x) -> typing.Iterable: + # Only a random subset of the features is monitored + if len(self.feature_indices) == 0: + self.feature_indices = self._sample_features(x, self.max_features) + + for att_id in self.feature_indices: + # First check if the feature is available + if att_id in x: + yield att_id, x[att_id] + + def _sample_features(self, x, max_features): + return self._rng.sample(list(x.keys()), max_features) + + +class ETLeafMajorityClass(ETLeaf, LeafMajorityClass): + def __init__(self, stats, depth, splitter, max_features, seed, **kwargs): + super().__init__(stats, depth, splitter, max_features, seed, **kwargs) + + +class ETLeafMean(ETLeaf, LeafMean): + def __init__(self, stats, depth, splitter, max_features, seed, **kwargs): + super().__init__(stats, depth, splitter, max_features, seed, **kwargs) + + +class ETLeafModel(ETLeaf, LeafModel): + def __init__(self, stats, depth, splitter, max_features, seed, **kwargs): + super().__init__(stats, depth, splitter, max_features, seed, **kwargs) + + +class ETLeafAdaptive(ETLeaf, LeafAdaptive): + def __init__(self, stats, depth, splitter, max_features, seed, **kwargs): + super().__init__(stats, depth, splitter, max_features, seed, **kwargs) diff --git a/river_extra/tree/splitter/__init__.py b/river_extra/tree/splitter/__init__.py new file mode 100644 index 0000000..c4b7713 --- /dev/null +++ b/river_extra/tree/splitter/__init__.py @@ -0,0 +1,3 @@ +from .random_splitter import ClassRandomSplitter, RegRandomSplitter + +__all__ = ["ClassRandomSplitter", "RegRandomSplitter"] diff --git a/river_extra/tree/splitter/random_splitter.py b/river_extra/tree/splitter/random_splitter.py new file mode 100644 index 0000000..dae63c5 --- /dev/null +++ b/river_extra/tree/splitter/random_splitter.py @@ -0,0 +1,93 @@ +import abc +import collections +import random +import sys + +from river import stats +from river.tree.splitter import Splitter +from river.tree.utils import BranchFactory + + +class RandomSplitter(Splitter): + def __init__(self, seed, buffer_size): + super().__init__() + self.seed = seed + self.buffer_size = buffer_size + + self.threshold = None + self.stats = None + + self._rng = random.Random(self.seed) + self._buffer = [] + + def __deepcopy__(self, memo): + """Change the behavior of deepcopy to allow copies have a different rng.""" + + seed = self._rng.randint(0, sys.maxsize) + new = self.__class__(seed=seed, buffer_size=self.buffer_size) + + return new + + @abc.abstractmethod + def _update_stats(self, branch, target_val, sample_weight): + pass + + def cond_proba(self, att_val, class_val) -> float: + """This attribute observer does not support probability density estimation.""" + raise NotImplementedError + + def update(self, att_val, target_val, sample_weight) -> "Splitter": + if self.threshold is None: + if len(self._buffer) < self.buffer_size: + self._buffer.append((att_val, target_val, sample_weight)) + return self + + mn = min(self._buffer, key=lambda t: t[0])[0] + mx = max(self._buffer, key=lambda t: t[0])[0] + + self.threshold = self._rng.uniform(mn, mx) + + for a, t, w in self._buffer: + self._update_stats(0 if a <= self.threshold else 1, t, w) + self._buffer = None + + return self + + self._update_stats(0 if att_val <= self.threshold else 1, target_val, sample_weight) + + return self + + def best_evaluated_split_suggestion(self, criterion, pre_split_dist, att_idx, binary_only): + post_split_dist = [self.stats[0], self.stats[1]] + merit = criterion.merit_of_split(pre_split_dist, post_split_dist) + + split_suggestion = BranchFactory( + merit=merit, + feature=att_idx, + split_info=self.threshold, + children_stats=post_split_dist, + ) + + return split_suggestion + + +class ClassRandomSplitter(RandomSplitter): + def __init__(self, seed, buffer_size): + super().__init__(seed, buffer_size) + self.stats = {0: collections.Counter(), 1: collections.Counter()} + + def _update_stats(self, branch, target_val, sample_weight): + self.stats[branch].update({target_val: sample_weight}) + + +class RegRandomSplitter(RandomSplitter): + def __init__(self, seed, buffer_size): + super().__init__(seed, buffer_size) + self.stats = {0: stats.Var(), 1: stats.Var()} + + def _update_stats(self, branch, target_val, sample_weight): + self.stats[branch].update(target_val, sample_weight) + + @property + def is_target_class(self) -> bool: + return False From 9721937064267e90ae9a1f48a2f72f33be579dcf Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Sun, 6 Nov 2022 10:25:08 -0300 Subject: [PATCH 29/50] add reference --- river_extra/ensemble/online_extra_trees.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/river_extra/ensemble/online_extra_trees.py b/river_extra/ensemble/online_extra_trees.py index e6f9991..1cb8c10 100644 --- a/river_extra/ensemble/online_extra_trees.py +++ b/river_extra/ensemble/online_extra_trees.py @@ -470,7 +470,7 @@ def _new_learning_node(self, initial_stats=None, parent=None): # noqa class ExtraTreesRegressor(ExtraTrees, base.Regressor): """Online Extra Trees regressor. - The online Extra Trees ensemble takes some steps further into randomization when + The online Extra Trees[^1] ensemble takes some steps further into randomization when compared to Adaptive Random Forests (ARF). A subspace of the feature space is considered at each split attempt, as ARF does, and online bagging or subbagging can also be (optionally) used. Nonetheless, Extra Trees randomizes the split candidates evaluated by each @@ -609,6 +609,11 @@ class ExtraTreesRegressor(ExtraTrees, base.Regressor): >>> evaluate.progressive_val_score(dataset, model, metric) RMSE: 3.24238 + References + ---------- + Mastelini, S. M., Nakano, F. K., Vens, C., & de Leon Ferreira, A. C. P. (2022). + Online Extra Trees Regressor. IEEE Transactions on Neural Networks and Learning Systems. + """ def __init__( From 348d03ec96810a1859cc784d665870f188667d58 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Sun, 6 Nov 2022 10:26:41 -0300 Subject: [PATCH 30/50] add reference --- river_extra/ensemble/online_extra_trees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/river_extra/ensemble/online_extra_trees.py b/river_extra/ensemble/online_extra_trees.py index 1cb8c10..c1b5460 100644 --- a/river_extra/ensemble/online_extra_trees.py +++ b/river_extra/ensemble/online_extra_trees.py @@ -611,7 +611,7 @@ class ExtraTreesRegressor(ExtraTrees, base.Regressor): References ---------- - Mastelini, S. M., Nakano, F. K., Vens, C., & de Leon Ferreira, A. C. P. (2022). + [^1]: Mastelini, S. M., Nakano, F. K., Vens, C., & de Leon Ferreira, A. C. P. (2022). Online Extra Trees Regressor. IEEE Transactions on Neural Networks and Learning Systems. """ From ab6aa9f53642a04dc1eea2a546d18bc87545c4c0 Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Sun, 6 Nov 2022 10:34:30 -0300 Subject: [PATCH 31/50] bump version --- river_extra/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/river_extra/__version__.py b/river_extra/__version__.py index e5ecb06..69ef906 100644 --- a/river_extra/__version__.py +++ b/river_extra/__version__.py @@ -1,3 +1,3 @@ -VERSION = (0, 13, 0) +VERSION = (0, 14, 0) __version__ = ".".join(map(str, VERSION)) From 723fd16201e3832303f2b32cf86f69e614db022b Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Sun, 6 Nov 2022 10:55:14 -0300 Subject: [PATCH 32/50] now for real --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c1db5b7..759ad64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] name = "river_extra" -version = "0.13.0" +version = "0.14.0" description = "Additional estimators for the River package" authors = ["MaxHalford "] license = "BSD-3" [tool.poetry.dependencies] python = ">=3.6.1,<4.0" -river = "0.13.0" +river = "0.14.0" [tool.poetry.dev-dependencies] From 2232b2c539365b96d3f92b85f9efeee8e3f2ad1e Mon Sep 17 00:00:00 2001 From: Saulo Martiello Mastelini Date: Thu, 22 Dec 2022 11:15:54 -0300 Subject: [PATCH 33/50] Housekeeping, simplify logic, and improve docs --- .../model_selection/regression_test.py | 5 +- river_extra/model_selection/sspt.py | 187 +++++++++--------- 2 files changed, 101 insertions(+), 91 deletions(-) diff --git a/river_extra/model_selection/regression_test.py b/river_extra/model_selection/regression_test.py index 31ad93b..d69149a 100644 --- a/river_extra/model_selection/regression_test.py +++ b/river_extra/model_selection/regression_test.py @@ -14,7 +14,7 @@ baseline_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) baseline_metric_plt = [] baseline_rolling_metric_plt = [] -baseline = preprocessing.AdaptiveStandardScaler() | linear_model.LinearRegression() +baseline = preprocessing.StandardScaler() | linear_model.LinearRegression() # SSPT - model and metric sspt_metric = metrics.RMSE() @@ -22,11 +22,10 @@ sspt_metric_plt = [] sspt_rolling_metric_plt = [] sspt = model_selection.SSPT( - estimator=preprocessing.AdaptiveStandardScaler() | linear_model.LinearRegression(), + estimator=preprocessing.StandardScaler() | linear_model.LinearRegression(), metric=metrics.RMSE(), grace_period=100, params_range={ - "AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, "LinearRegression": { "l2": (float, (0.0, 0.0001)), "optimizer": {"lr": {"learning_rate": (float, (0.009, 0.011))}}, diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index b76e15e..61c010d 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -86,123 +86,133 @@ def __init__( elif isinstance(border, anomaly.base.AnomalyDetector): self._scorer_name = "classify" - def __generate(self, p_data) -> numbers.Number: - p_type, p_range = p_data - if p_type == int: - return self._rng.randint(p_range[0], p_range[1]) - elif p_type == float: - return self._rng.uniform(p_range[0], p_range[1]) + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + return self._rng.randint(hp_range[0], hp_range[1]) + elif hp_type == float: + return self._rng.uniform(hp_range[0], hp_range[1]) - def __combine(self, p_info, param1, param2, func): - - p_type, p_range = p_info - new_val = func(param1, param2) + def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + new_val = func(hp_est_1, hp_est_2) # Range sanity checks - if new_val < p_range[0]: - new_val = p_range[0] - if new_val > p_range[1]: - new_val = p_range[1] + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] - new_val = round(new_val, 0) if p_type == int else new_val + new_val = round(new_val, 0) if hp_type == int else new_val return new_val - def __flatten(self, prefix, scaled, p_info, e_info): - _, p_range = p_info - interval = p_range[1] - p_range[0] - scaled[prefix] = (e_info - p_range[0]) / interval + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ - def _recurse_params( - self, operation, p_data, e1_data, *, func=None, e2_data=None, prefix=None, scaled=None - ): # Sub-component needs to be instantiated - if isinstance(e1_data, tuple): - sub_class, sub_data1 = e1_data + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 if operation == "combine": - _, sub_data2 = e2_data + est_2 = est_2[1] else: - sub_data2 = {} + est_2 = {} sub_config = {} - for sub_param, sub_info in p_data.items(): + for sub_hp_name, sub_hp_data in hp_data.items(): if operation == "scale": - sub_prefix = prefix + "__" + sub_param + sub_hp_prefix = hp_prefix + "__" + sub_hp_name else: - sub_prefix = None + sub_hp_prefix = None - sub_config[sub_param] = self._recurse_params( + sub_config[sub_hp_name] = self._traverse_hps( operation=operation, - p_data=sub_info, - e1_data=sub_data1[sub_param], + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], func=func, - e2_data=sub_data2.get(sub_param, None), - prefix=sub_prefix, - scaled=scaled, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, ) return sub_class(**sub_config) # We reached the numeric parameters - if isinstance(p_data, tuple): + if isinstance(est_1, numbers.Number): if operation == "generate": - return self.__generate(p_data) + return self.__generate(hp_data) if operation == "scale": - self.__flatten(prefix, scaled, p_data, e1_data) + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) return # combine - return self.__combine(p_data, e1_data, e2_data, func) + return self.__combine(hp_data, est_1, est_2, func) # The sub-parameters need to be expanded config = {} - for p_name, p_info in p_data.items(): - e1_info = e1_data[p_name] + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] if operation == "combine": - e2_info = e2_data[p_name] + sub_est_2 = est_2[sub_hp_name] else: - e2_info = {} + sub_est_2 = {} if operation == "scale": - sub_prefix = prefix + "__" + p_name if len(prefix) > 0 else p_name + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name else: - sub_prefix = None + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) - if not isinstance(p_info, dict): - config[p_name] = self._recurse_params( - operation=operation, - p_data=p_info, - e1_data=e1_info, - func=func, - e2_data=e2_info, - prefix=sub_prefix, - scaled=scaled, - ) - else: - sub_config = {} - for sub_name, sub_info in p_info.items(): - - if operation == "scale": - sub_prefix2 = sub_prefix + "__" + sub_name - else: - sub_prefix2 = None - - sub_config[sub_name] = self._recurse_params( - operation=operation, - p_data=sub_info, - e1_data=e1_info[sub_name], - func=func, - e2_data=e2_info.get(sub_name, None), - prefix=sub_prefix2, - scaled=scaled, - ) - config[p_name] = sub_config return config def _random_config(self): - return self._recurse_params( + return self._traverse_hps( operation="generate", - p_data=self.params_range, - e1_data=self.estimator._get_params() + hp_data=self.params_range, + est_1=self.estimator._get_params() ) def _create_simplex(self, model) -> typing.List: @@ -237,16 +247,17 @@ def _sort_simplex(self): def _gen_new_estimator(self, e1, e2, func): """Generate new configuration given two estimators and a combination function.""" - e1_p = e1.estimator._get_params() - e2_p = e2.estimator._get_params() + est_1_hps = e1.estimator._get_params() + est_2_hps = e2.estimator._get_params() - new_config = self._recurse_params( + new_config = self._traverse_hps( operation="combine", - p_data=self.params_range, - e1_data=e1_p, + hp_data=self.params_range, + est_1=est_1_hps, func=func, - e2_data=e2_p + est_2=est_2_hps ) + # Modify the current best contender with the new hyperparameter values new = ModelWrapper( copy.deepcopy(self._simplex[0].estimator), @@ -322,12 +333,12 @@ def _nelder_mead_operators(self): def _normalize_flattened_hyperspace(self, orig): scaled = {} - self._recurse_params( + self._traverse_hps( operation="scale", - p_data=self.params_range, - e1_data=orig, - prefix="", - scaled=scaled + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled ) return scaled From daf6e06f1f7a15a9e9f5d197282e36158baf2b70 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Fri, 30 Dec 2022 11:31:39 +0000 Subject: [PATCH 34/50] v0.2 testing --- .DS_Store | Bin 0 -> 6148 bytes river_extra/.DS_Store | Bin 0 -> 6148 bytes .../model_selection/classification_test.py | 7 +++- .../model_selection/regression_test.py | 13 +++---- river_extra/model_selection/teste.py | 35 ++++++++++++++++++ 5 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 .DS_Store create mode 100644 river_extra/.DS_Store create mode 100644 river_extra/model_selection/teste.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..7b71020659ba094ace3f65d1b90e045382db4ec3 GIT binary patch literal 6148 zcmeHKK~BRk5FD3EZB+^N$OS1ch{O-lk{&7n2l@b@DF+}B6riHF+<6SI;W3W1L`)=`1g@Mc&G` zz&0u%YuCdOW;nzYU)HZ!utz?NF+Kk9#u_lhJqldn5hd1$k+YTIMpu2qdTp?F$y(D( zW?z$?kss3w+#;`Y-F|~Rz$u??j1z(xVtA0oP760lPA zTL%k&1t4}A_QtyWGzupQm;|g8*+X+mB`Vd0D~3}#<59><0#=GjhYObv7iMS71k*BRT)4Ki~iBBx{)hrog{az;)AkI^ipYv$geda@I!lbGn%1 nRf-!5EB+{Et{laCbZ?ADQXwV*D@8_V_9NhBu)-AhQw2T&B_|T?$Bn>r=qL4~_2F6;6rq>EIA8 z0CB-^80XPT5Ss^xUE!3-2+fj8OsZ9jVM%AaRbE#(B_07 zC)|3 k6Qdk+;pO-~k}|LPocmqjlo)izgHF`Xfa@ZY0{^YRH#Djjng9R* literal 0 HcmV?d00001 diff --git a/river_extra/model_selection/classification_test.py b/river_extra/model_selection/classification_test.py index b8358a1..ed817a5 100644 --- a/river_extra/model_selection/classification_test.py +++ b/river_extra/model_selection/classification_test.py @@ -31,8 +31,11 @@ metric=sspt_rolling_metric, grace_period=100, params_range={ - "delta": (float, (0.00001, 0.0001)), - "grace_period": (int, (100, 500)), + "AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, + "HoeffdingTreeClassifier": { + "delta": (float, (0.00001, 0.0001)), + "grace_period": (int, (100, 500)), + }, }, drift_input=lambda yt, yp: 0 if yt == yp else 1, convergence_sphere=0.000001, diff --git a/river_extra/model_selection/regression_test.py b/river_extra/model_selection/regression_test.py index 31ad93b..0353995 100644 --- a/river_extra/model_selection/regression_test.py +++ b/river_extra/model_selection/regression_test.py @@ -1,5 +1,5 @@ import matplotlib.pyplot as plt -from river import datasets, drift, linear_model, metrics, preprocessing, utils +from river import datasets, drift, linear_model, metrics, preprocessing, utils, rules from river_extra import model_selection @@ -14,7 +14,7 @@ baseline_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) baseline_metric_plt = [] baseline_rolling_metric_plt = [] -baseline = preprocessing.AdaptiveStandardScaler() | linear_model.LinearRegression() +baseline = preprocessing.AdaptiveStandardScaler() | rules.AMRules() # SSPT - model and metric sspt_metric = metrics.RMSE() @@ -22,15 +22,14 @@ sspt_metric_plt = [] sspt_rolling_metric_plt = [] sspt = model_selection.SSPT( - estimator=preprocessing.AdaptiveStandardScaler() | linear_model.LinearRegression(), + estimator=preprocessing.AdaptiveStandardScaler() | rules.AMRules(), metric=metrics.RMSE(), grace_period=100, params_range={ "AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, - "LinearRegression": { - "l2": (float, (0.0, 0.0001)), - "optimizer": {"lr": {"learning_rate": (float, (0.009, 0.011))}}, - "intercept_lr": {"learning_rate": (float, (0.009, 0.011))}, + "AMRules": { + "delta": (float, (0.005, 0.02)), + "n_min": (int, (50, 500)) }, }, drift_input=lambda yt, yp: abs(yt - yp), diff --git a/river_extra/model_selection/teste.py b/river_extra/model_selection/teste.py new file mode 100644 index 0000000..a2ffa13 --- /dev/null +++ b/river_extra/model_selection/teste.py @@ -0,0 +1,35 @@ +from scipy.stats import qmc + +sampler = qmc.LatinHypercube(d=4) + +sample = sampler.random(n=5) + +print(sample) + +import numpy as np +import matplotlib.pyplot as plt + +from smt.sampling_methods import LHS + +xlimits = np.array([[0.0, 100.0], [0.0, 1.0], [20.0, 30.0]]) +sampling = LHS(xlimits=xlimits) + +num = 50 +x = sampling(num) + +print(x.shape) +plt.plot(x[:, 0], x[:, 1], "o") +plt.xlabel("x") +plt.ylabel("y") +plt.show() +fig = plt.figure() +ax = fig.add_subplot(projection='3d') +for val in x: + print(val) + ax.scatter(val[0], val[1], val[2], marker='o') + +ax.set_xlabel('X Label') +ax.set_ylabel('Y Label') +ax.set_zlabel('Z Label') + +plt.show() From 0ed4481c659ed8528da0a7c603df48353de649b2 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Fri, 24 Feb 2023 14:11:39 +0000 Subject: [PATCH 35/50] v1.0 --- river_extra/model_selection/.DS_Store | Bin 0 -> 6148 bytes .../model_selection/classification_test.py | 80 ------------------ .../model_selection/clustering_test.py | 72 ---------------- .../model_selection/recommendation_test.py | 80 ------------------ .../model_selection/regression_test.py | 78 ----------------- river_extra/model_selection/sspt.py | 78 +++++++++++++++++ river_extra/model_selection/teste.py | 35 -------- 7 files changed, 78 insertions(+), 345 deletions(-) create mode 100644 river_extra/model_selection/.DS_Store delete mode 100644 river_extra/model_selection/classification_test.py delete mode 100644 river_extra/model_selection/clustering_test.py delete mode 100644 river_extra/model_selection/recommendation_test.py delete mode 100644 river_extra/model_selection/regression_test.py delete mode 100644 river_extra/model_selection/teste.py diff --git a/river_extra/model_selection/.DS_Store b/river_extra/model_selection/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Fri, 24 Feb 2023 14:29:00 +0000 Subject: [PATCH 36/50] v1.01 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 44eccf2..0e743cf 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ poetry.lock *.whl *.tar.gz *.pyc +*.DS_Store From 42bda7810f6eb691c8fcdf8a36da6e279c92d395 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Fri, 24 Feb 2023 14:31:58 +0000 Subject: [PATCH 37/50] v1.02 --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0e743cf..6c6a95b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ poetry.lock *.whl *.tar.gz *.pyc -*.DS_Store +.DS_Store From 0efde7dc1e6714a39ce0c7380dd35ab610f526ba Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Fri, 24 Feb 2023 14:31:58 +0000 Subject: [PATCH 38/50] v1.02 --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0e743cf..6c6a95b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ poetry.lock *.whl *.tar.gz *.pyc -*.DS_Store +.DS_Store From 367dbe7e45f087f8277f093660ef97cd58f184e9 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Fri, 24 Feb 2023 14:35:10 +0000 Subject: [PATCH 39/50] v1.03 --- .DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 7b71020659ba094ace3f65d1b90e045382db4ec3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKK~BRk5FD3EZB+^N$OS1ch{O-lk{&7n2l@b@DF+}B6riHF+<6SI;W3W1L`)=`1g@Mc&G` zz&0u%YuCdOW;nzYU)HZ!utz?NF+Kk9#u_lhJqldn5hd1$k+YTIMpu2qdTp?F$y(D( zW?z$?kss3w+#;`Y-F|~Rz$u??j1z(xVtA0oP760lPA zTL%k&1t4}A_QtyWGzupQm;|g8*+X+mB`Vd0D~3}#<59><0#=GjhYObv7iMS71k*BRT)4Ki~iBBx{)hrog{az;)AkI^ipYv$geda@I!lbGn%1 nRf-!5EB+{Et{laCbZ?ADQXwV*D@8_V_9NhBu)-AhQw2T&B Date: Fri, 24 Feb 2023 14:40:40 +0000 Subject: [PATCH 40/50] v1.04 --- river_extra/.DS_Store | Bin 6148 -> 0 bytes river_extra/model_selection/sspt.py | 41 ---------------------------- 2 files changed, 41 deletions(-) delete mode 100644 river_extra/.DS_Store diff --git a/river_extra/.DS_Store b/river_extra/.DS_Store deleted file mode 100644 index c869a64bda33cef750f5af2e142de92e79e27bcf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKyG{c!5S)b+ktmXq(!anToTBgrd_WMn6i5eG1p2G;U3?m|4+%vFRYVibO6#%L zJGMN_|T?$Bn>r=qL4~_2F6;6rq>EIA8 z0CB-^80XPT5Ss^xUE!3-2+fj8OsZ9jVM%AaRbE#(B_07 zC)|3 k6Qdk+;pO-~k}|LPocmqjlo)izgHF`Xfa@ZY0{^YRH#Djjng9R* diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index d4b8384..1c26dca 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -59,11 +59,6 @@ def __init__( self.drift_detector = drift_detector self.convergence_sphere = convergence_sphere - self.vtau=[] - self.vdelta=[] - self.vgrace=[] - self.vscore=[] - self.seed = seed self._n = 0 @@ -303,74 +298,38 @@ def _nelder_mead_operators(self): scaled_params = list(self._normalize_flattened_hyperspace( self._simplex[0].estimator._get_params(), ).values()) - self.vtau.append(scaled_params[0]) - self.vdelta.append(scaled_params[1]) - self.vgrace.append(scaled_params[2]) - self.vscore.append(self._simplex[0].metric.get()) scaled_params = list(self._normalize_flattened_hyperspace( self._simplex[1].estimator._get_params(), ).values()) - self.vtau.append(scaled_params[0]) - self.vdelta.append(scaled_params[1]) - self.vgrace.append(scaled_params[2]) - self.vscore.append(self._simplex[1].metric.get()) scaled_params = list(self._normalize_flattened_hyperspace( self._simplex[2].estimator._get_params(), ).values()) - self.vtau.append(scaled_params[0]) - self.vdelta.append(scaled_params[1]) - self.vgrace.append(scaled_params[2]) - self.vscore.append(self._simplex[2].metric.get()) scaled_params = list(self._normalize_flattened_hyperspace( self._expanded["reflection"].estimator._get_params(), ).values()) - self.vtau.append(scaled_params[0]) - self.vdelta.append(scaled_params[1]) - self.vgrace.append(scaled_params[2]) - self.vscore.append(self._expanded["reflection"].metric.get()) scaled_params = list(self._normalize_flattened_hyperspace( self._expanded["contraction1"].estimator._get_params(), ).values()) - self.vtau.append(scaled_params[0]) - self.vdelta.append(scaled_params[1]) - self.vgrace.append(scaled_params[2]) - self.vscore.append(self._expanded["contraction1"].metric.get()) scaled_params = list(self._normalize_flattened_hyperspace( self._expanded["contraction2"].estimator._get_params(), ).values()) - self.vtau.append(scaled_params[0]) - self.vdelta.append(scaled_params[1]) - self.vgrace.append(scaled_params[2]) - self.vscore.append(self._expanded["contraction2"].metric.get()) scaled_params = list(self._normalize_flattened_hyperspace( self._expanded["expansion"].estimator._get_params(), ).values()) - self.vtau.append(scaled_params[0]) - self.vdelta.append(scaled_params[1]) - self.vgrace.append(scaled_params[2]) - self.vscore.append(self._expanded["expansion"].metric.get()) scaled_params = list(self._normalize_flattened_hyperspace( self._expanded["midpoint"].estimator._get_params(), ).values()) - self.vtau.append(scaled_params[0]) - self.vdelta.append(scaled_params[1]) - self.vgrace.append(scaled_params[2]) - self.vscore.append(self._expanded["midpoint"].metric.get()) scaled_params = list(self._normalize_flattened_hyperspace( self._expanded["shrink"].estimator._get_params(), ).values()) - self.vtau.append(scaled_params[0]) - self.vdelta.append(scaled_params[1]) - self.vgrace.append(scaled_params[2]) - self.vscore.append(self._expanded["shrink"].metric.get()) if c1.is_better_than(c2): self._expanded["contraction"] = self._expanded["contraction1"] From a1080357d83a96aca0f97cc401d3d3cac13873bc Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Fri, 24 Feb 2023 14:42:18 +0000 Subject: [PATCH 41/50] v1.1 --- river_extra/model_selection/.DS_Store | Bin 6148 -> 0 bytes river_extra/model_selection/sspt.py | 36 -------------------------- 2 files changed, 36 deletions(-) delete mode 100644 river_extra/model_selection/.DS_Store diff --git a/river_extra/model_selection/.DS_Store b/river_extra/model_selection/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Fri, 24 Feb 2023 15:05:32 +0000 Subject: [PATCH 42/50] Update .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6c6a95b..44eccf2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,3 @@ poetry.lock *.whl *.tar.gz *.pyc -.DS_Store From aebc8e914927ae65ae42a3aa697929ef8d1b9825 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Thu, 11 May 2023 16:17:21 +0100 Subject: [PATCH 43/50] V1.2 Added: Grid search Random search - option Latin Hypercube sample Hyper Band Functional Anova I need to refactor and optimize more code --- .../model_selection/functional_anova.py | 553 +++++++++++++++++ river_extra/model_selection/grid_search.py | 550 +++++++++++++++++ river_extra/model_selection/hyper_band.py | 437 ++++++++++++++ river_extra/model_selection/random_search.py | 555 ++++++++++++++++++ 4 files changed, 2095 insertions(+) create mode 100644 river_extra/model_selection/functional_anova.py create mode 100644 river_extra/model_selection/grid_search.py create mode 100644 river_extra/model_selection/hyper_band.py create mode 100644 river_extra/model_selection/random_search.py diff --git a/river_extra/model_selection/functional_anova.py b/river_extra/model_selection/functional_anova.py new file mode 100644 index 0000000..ada5c18 --- /dev/null +++ b/river_extra/model_selection/functional_anova.py @@ -0,0 +1,553 @@ +import statistics +from ast import operator +import collections +import copy +import math +import numbers +import random +import typing +import pandas as pd +import numpy as np +from itertools import combinations +from tqdm import tqdm + +import numpy as np + +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + +class Functional_Anova(base.Estimator): + """Bandit Self Parameter Tuning + + Parameters + ---------- + estimator + metric + params_range + drift_input + grace_period + drift_detector + convergence_sphere + seed + + References + ---------- + [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning + for data streams. Information Fusion, 76, 75-86. + + """ + + _START_RANDOM = "random" + _START_WARM = "warm" + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + nr_estimators: int = 50, + seed: int = None, + ): + super().__init__() + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + + self.nr_estimators = nr_estimators + + self.seed = seed + + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + grids = np.meshgrid(np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__]))) + self._sample=np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) + + self._df_hpi = pd.DataFrame(columns=['params', 'instances', 'score']) + + self._i=0 + self._p=0 + self._counter=0 + self._best_estimator = None + self._bandits = self._create_bandits(estimator,nr_estimators) + + # Convergence criterion + self._old_centroid = None + + self._pruned_configurations=[] + # Meta-programming + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" + + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return int (val) + elif hp_type == float: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return val + + def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def __combine_bee(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_est_1>=hp_est_2: + func=lambda h1, h2: h2 + ((h1 + h2)*9 / 10) + else: + func = lambda h1, h2: h1 + ((h1 + h2)*9 / 10) + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def analyze_fanova(self, cal_df, max_iter=-1): + metric='score' + params = list(eval(cal_df.loc[0, 'params']).keys()) + result = pd.DataFrame(cal_df.loc[:, metric].copy()) + tmp_df = cal_df.loc[:, 'params'].copy() + for key in params: + result.insert(loc=0, column=key, value=tmp_df.apply(lambda x: eval(x)[key])) + col_name = params[:] + col_name.append(metric) + cal_df = result.reindex(columns=col_name).copy() + + axis = ['id'] + axis.extend(list(cal_df.columns)) + params = axis[1:-1] + metric = axis[-1] + f = pd.DataFrame(columns=axis) + f.loc[0, :] = np.nan + f.loc[0, metric] = cal_df[metric].mean() + f.loc[0, 'id'] = hash(str([])) + v_all = np.std(cal_df[metric].to_numpy()) ** 2 + v = pd.DataFrame(columns=['u', 'v_u', 'F_u(v_u/v_all)']) + for k in range(1, len(params) + 1): + for u in combinations(params, k): + ne_u = set(params) - set(u) + a_u = cal_df.groupby(list(u)).mean().reset_index() + if len(ne_u): + for nu in ne_u: + a_u.loc[:, nu] = np.nan + col_name = cal_df.columns.tolist() + a_u = a_u.reindex(columns=col_name) + sum_f_w = pd.DataFrame(columns=f.columns[:]) + tmp = [] + w_list = [] + for i in range(len(u)): + tmp.extend(list(combinations(u, i))) + for t in tmp: + w_list.append(list(t)) + + for w in w_list: + sum_f_w = pd.concat([sum_f_w, f[f['id'] == hash(str(w))]], ignore_index=True) + col_name = sum_f_w.columns.tolist() + a_u = a_u.reindex(columns=col_name) + for row_index, r in sum_f_w.iterrows(): + r2 = r[1:-1] + not_null_index = r2.notnull().values + if not not_null_index.any(): + a_u.loc[:, col_name[-1]] -= r[-1] + else: + left, right = a_u.iloc[:, 1:-1].align(r2[not_null_index], axis=1, copy=False) + equal_index = (left == right).values.sum(axis=1) + a_u.loc[equal_index == not_null_index.sum(), col_name[-1]] -= r[-1] + a_u['id'] = hash(str(list(u))) + f = pd.concat([f, a_u], ignore_index=True) + tmp_f_u = a_u.loc[:, metric].to_numpy() + v = pd.concat([v, pd.DataFrame( + [{'u': u, 'v_u': (tmp_f_u ** 2).mean(), 'F_u(v_u/v_all)': (tmp_f_u ** 2).mean() / v_all}])], + ignore_index=True) + + return v + + + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ + + # Sub-component needs to be instantiated + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 + + if operation == "combine": + est_2 = est_2[1] + else: + est_2 = {} + + sub_config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name + else: + sub_hp_prefix = None + + sub_config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], + func=func, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(est_1, numbers.Number): + if operation == "generate": + return self.__generate(hp_data) + if operation == "scale": + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) + return + # combine + return self.__combine(hp_data, est_1, est_2, func) + + # The sub-parameters need to be expanded + config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] + + if operation == "combine": + sub_est_2 = est_2[sub_hp_name] + else: + sub_est_2 = {} + + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name + else: + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + + return config + + def _random_config(self): + return self._traverse_hps( + operation="generate", + hp_data=self.params_range, + est_1=self.estimator._get_params(), + ) + + def _create_bandits(self, model, nr_estimators) -> typing.List: + bandits = [None] * nr_estimators + + for i in range(nr_estimators): + bandits[i] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True), + ) + self._i+=1 + self._p=0 + self._i=0 + return bandits + + def _sort_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._bandits.sort(key=lambda mw: mw.metric.get()) + + def _prune_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + self._bandits[0].estimator._get_params() + half = int(1*len(self._bandits) / 10) + for i in range(len(self._bandits)): + #if i>half: + lst=list(self.params_range[self.estimator[1].__class__.__name__].keys()) + mydict={} + for x in lst: + mydict[x]=self._bandits[i].estimator._get_params()[self.estimator[1].__class__.__name__][x] + #mydict['instances'] + self._df_hpi.loc[len(self._df_hpi.index)] = [str(mydict), self._bandits[i].estimator[self.estimator[1].__class__.__name__].summary['total_observed_weight'], self._bandits[i].metric.get()] + #'{\'hp1\': ' + str(row[1]['hp1']) + ', \'hp2\': ' + str(row[1]['hp2']) + ', \'hp3\': ' + str(row[1]['hp3']) + '}' + #new_col = [] + #for row in df.iterrows(): + # new_col.append( + # ) + #df['params'] = new_col + self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ + str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) + #else: + #self._pruned_configurations.append( + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ + # 'grace_period']) + ',' + + # str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ + # 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) + + #if len(self._bandits)-half>2: + # self._bandits=self._bandits[:-half] + #else: + # self._bandits=self._bandits[:-(len(self._bandits)-3)] + + + def _gen_new_estimator(self, e1, e2, func): + """Generate new configuration given two estimators and a combination function.""" + + est_1_hps = e1.estimator._get_params() + est_2_hps = e2.estimator._get_params() + + new_config = self._traverse_hps( + operation="combine", + hp_data=self.params_range, + est_1=est_1_hps, + func=func, + est_2=est_2_hps + ) + + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(self._bandits[0].estimator), + self.metric.clone(include_attributes=True), + ) + new.estimator.mutate(new_config) + + return new + + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._traverse_hps( + operation="scale", + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled + ) + return scaled + + @property + def _models_converged(self) -> bool: + if len(self._bandits)==3: + return True + else: + return False + + def _learn_converged(self, x, y): + scorer = getattr(self._best_estimator, self._scorer_name) + y_pred = scorer(x) + + input = self.drift_input(y, y_pred) + self.drift_detector.update(input) + + # We need to start the optimization process from scratch + if self.drift_detector.drift_detected: + self._n = 0 + self._converged = False + self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) + + # There is no proven best model right now + self._best_estimator = None + return + + self._best_estimator.learn_one(x, y) + + def _learn_not_converged(self, x, y): + for wrap in self._bandits: + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) + wrap.metric.update(y, y_pred) + wrap.estimator.learn_one(x, y) + + # Keep the simplex ordered + self._sort_bandits() + + if self._n == self.grace_period: + self._n = 0 + self._prune_bandits() + df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] + df3 = df3.reset_index() + importance=self.analyze_fanova(df3) + print(importance) + #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] + + #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] + print("Nr bandits: ",len(self._bandits)) + # 1. Simplex in sphere + scaled_params_b = self._normalize_flattened_hyperspace( + self._bandits[0].estimator._get_params(), + ) + scaled_params_g = self._normalize_flattened_hyperspace( + self._bandits[1].estimator._get_params(), + ) + scaled_params_w = self._normalize_flattened_hyperspace( + self._bandits[2].estimator._get_params(), + ) + #for i in range(1,len(self._bandits)): + # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) + + print("----------") + print( + "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric + ) + print( + "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric + ) + print( + "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric + ) + hyper_points = [ + list(scaled_params_b.values()), + list(scaled_params_g.values()), + list(scaled_params_w.values()), + ] + vectors = np.array(hyper_points) + self._old_centroid = dict( + zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) + ) + + + if self._models_converged: + self._converged = True + self._best_estimator = self._bandits[0].estimator + + def learn_one(self, x, y): + self._n += 1 + self._counter += 1 + + if self.converged: + self._learn_converged(x, y) + else: + self._learn_not_converged(x, y) + + return self + + @property + def best(self): + if not self._converged: + # Lazy selection of the best model + self._sort_bandits() + return self._bandits[0].estimator + + return self._best_estimator + + @property + def converged(self): + return self._converged + + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) + + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) + + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) + + def debug_one(self, x, **kwargs): + try: + return self.best.debug_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) diff --git a/river_extra/model_selection/grid_search.py b/river_extra/model_selection/grid_search.py new file mode 100644 index 0000000..64a69b2 --- /dev/null +++ b/river_extra/model_selection/grid_search.py @@ -0,0 +1,550 @@ +import statistics +from ast import operator +import collections +import copy +import math +import numbers +import random +import typing +import pandas as pd +import numpy as np +from itertools import combinations +from tqdm import tqdm + +import numpy as np + +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + +class Grid_Search(base.Estimator): + """Bandit Self Parameter Tuning + + Parameters + ---------- + estimator + metric + params_range + drift_input + grace_period + drift_detector + convergence_sphere + seed + + References + ---------- + [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning + for data streams. Information Fusion, 76, 75-86. + + """ + + _START_RANDOM = "random" + _START_WARM = "warm" + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + nr_estimators: int = 50, + seed: int = None, + ): + super().__init__() + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + + self.nr_estimators = nr_estimators + + self.seed = seed + + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + grids = np.meshgrid(np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__]))) + self._sample=np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) + + self._i=0 + self._p=0 + self._counter=0 + self._best_estimator = None + self._bandits = self._create_bandits(estimator,nr_estimators) + + # Convergence criterion + self._old_centroid = None + + # Meta-programming + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" + + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return int (val) + elif hp_type == float: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return val + + def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def __combine_bee(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_est_1>=hp_est_2: + func=lambda h1, h2: h2 + ((h1 + h2)*9 / 10) + else: + func = lambda h1, h2: h1 + ((h1 + h2)*9 / 10) + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def analyze_fanova(self, cal_df, max_iter=-1): + metric='score' + params = list(eval(cal_df.loc[0, 'params']).keys()) + result = pd.DataFrame(cal_df.loc[:, metric].copy()) + tmp_df = cal_df.loc[:, 'params'].copy() + for key in params: + result.insert(loc=0, column=key, value=tmp_df.apply(lambda x: eval(x)[key])) + col_name = params[:] + col_name.append(metric) + cal_df = result.reindex(columns=col_name).copy() + + axis = ['id'] + axis.extend(list(cal_df.columns)) + params = axis[1:-1] + metric = axis[-1] + f = pd.DataFrame(columns=axis) + f.loc[0, :] = np.nan + f.loc[0, metric] = cal_df[metric].mean() + f.loc[0, 'id'] = hash(str([])) + v_all = np.std(cal_df[metric].to_numpy()) ** 2 + v = pd.DataFrame(columns=['u', 'v_u', 'F_u(v_u/v_all)']) + for k in range(1, len(params) + 1): + for u in combinations(params, k): + ne_u = set(params) - set(u) + a_u = cal_df.groupby(list(u)).mean().reset_index() + if len(ne_u): + for nu in ne_u: + a_u.loc[:, nu] = np.nan + col_name = cal_df.columns.tolist() + a_u = a_u.reindex(columns=col_name) + sum_f_w = pd.DataFrame(columns=f.columns[:]) + tmp = [] + w_list = [] + for i in range(len(u)): + tmp.extend(list(combinations(u, i))) + for t in tmp: + w_list.append(list(t)) + + for w in w_list: + sum_f_w = pd.concat([sum_f_w, f[f['id'] == hash(str(w))]], ignore_index=True) + col_name = sum_f_w.columns.tolist() + a_u = a_u.reindex(columns=col_name) + for row_index, r in sum_f_w.iterrows(): + r2 = r[1:-1] + not_null_index = r2.notnull().values + if not not_null_index.any(): + a_u.loc[:, col_name[-1]] -= r[-1] + else: + left, right = a_u.iloc[:, 1:-1].align(r2[not_null_index], axis=1, copy=False) + equal_index = (left == right).values.sum(axis=1) + a_u.loc[equal_index == not_null_index.sum(), col_name[-1]] -= r[-1] + a_u['id'] = hash(str(list(u))) + f = pd.concat([f, a_u], ignore_index=True) + tmp_f_u = a_u.loc[:, metric].to_numpy() + v = pd.concat([v, pd.DataFrame( + [{'u': u, 'v_u': (tmp_f_u ** 2).mean(), 'F_u(v_u/v_all)': (tmp_f_u ** 2).mean() / v_all}])], + ignore_index=True) + + return v + + + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ + + # Sub-component needs to be instantiated + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 + + if operation == "combine": + est_2 = est_2[1] + else: + est_2 = {} + + sub_config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name + else: + sub_hp_prefix = None + + sub_config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], + func=func, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(est_1, numbers.Number): + if operation == "generate": + return self.__generate(hp_data) + if operation == "scale": + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) + return + # combine + return self.__combine(hp_data, est_1, est_2, func) + + # The sub-parameters need to be expanded + config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] + + if operation == "combine": + sub_est_2 = est_2[sub_hp_name] + else: + sub_est_2 = {} + + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name + else: + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + + return config + + def _random_config(self): + return self._traverse_hps( + operation="generate", + hp_data=self.params_range, + est_1=self.estimator._get_params(), + ) + + def _create_bandits(self, model, nr_estimators) -> typing.List: + bandits = [None] * nr_estimators + + for i in range(nr_estimators): + bandits[i] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True), + ) + self._i+=1 + self._p=0 + self._i=0 + return bandits + + def _sort_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._bandits.sort(key=lambda mw: mw.metric.get()) + + def _prune_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + self._bandits[0].estimator._get_params() + half = int(1*len(self._bandits) / 10) + for i in range(len(self._bandits)): + #if i>half: + lst=list(self.params_range[self.estimator[1].__class__.__name__].keys()) + mydict={} + for x in lst: + mydict[x]=self._bandits[i].estimator._get_params()[self.estimator[1].__class__.__name__][x] + #mydict['instances'] + self._df_hpi.loc[len(self._df_hpi.index)] = [str(mydict), self._bandits[i].estimator[self.estimator[1].__class__.__name__].summary['total_observed_weight'], self._bandits[i].metric.get()] + #'{\'hp1\': ' + str(row[1]['hp1']) + ', \'hp2\': ' + str(row[1]['hp2']) + ', \'hp3\': ' + str(row[1]['hp3']) + '}' + #new_col = [] + #for row in df.iterrows(): + # new_col.append( + # ) + #df['params'] = new_col + self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ + str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) + #else: + #self._pruned_configurations.append( + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ + # 'grace_period']) + ',' + + # str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ + # 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) + + #if len(self._bandits)-half>2: + # self._bandits=self._bandits[:-half] + #else: + # self._bandits=self._bandits[:-(len(self._bandits)-3)] + + + def _gen_new_estimator(self, e1, e2, func): + """Generate new configuration given two estimators and a combination function.""" + + est_1_hps = e1.estimator._get_params() + est_2_hps = e2.estimator._get_params() + + new_config = self._traverse_hps( + operation="combine", + hp_data=self.params_range, + est_1=est_1_hps, + func=func, + est_2=est_2_hps + ) + + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(self._bandits[0].estimator), + self.metric.clone(include_attributes=True), + ) + new.estimator.mutate(new_config) + + return new + + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._traverse_hps( + operation="scale", + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled + ) + return scaled + + @property + def _models_converged(self) -> bool: + if len(self._bandits)==3: + return True + else: + return False + + def _learn_converged(self, x, y): + scorer = getattr(self._best_estimator, self._scorer_name) + y_pred = scorer(x) + + input = self.drift_input(y, y_pred) + self.drift_detector.update(input) + + # We need to start the optimization process from scratch + if self.drift_detector.drift_detected: + self._n = 0 + self._converged = False + self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) + + # There is no proven best model right now + self._best_estimator = None + return + + self._best_estimator.learn_one(x, y) + + def _learn_not_converged(self, x, y): + for wrap in self._bandits: + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) + wrap.metric.update(y, y_pred) + wrap.estimator.learn_one(x, y) + + # Keep the simplex ordered + self._sort_bandits() + + if self._n == self.grace_period: + self._n = 0 + self._prune_bandits() + df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] + df3 = df3.reset_index() + importance=self.analyze_fanova(df3) + print(importance) + #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] + + #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] + print("Nr bandits: ",len(self._bandits)) + # 1. Simplex in sphere + scaled_params_b = self._normalize_flattened_hyperspace( + self._bandits[0].estimator._get_params(), + ) + scaled_params_g = self._normalize_flattened_hyperspace( + self._bandits[1].estimator._get_params(), + ) + scaled_params_w = self._normalize_flattened_hyperspace( + self._bandits[2].estimator._get_params(), + ) + #for i in range(1,len(self._bandits)): + # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) + + print("----------") + print( + "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric + ) + print( + "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric + ) + print( + "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric + ) + hyper_points = [ + list(scaled_params_b.values()), + list(scaled_params_g.values()), + list(scaled_params_w.values()), + ] + vectors = np.array(hyper_points) + self._old_centroid = dict( + zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) + ) + + + if self._models_converged: + self._converged = True + self._best_estimator = self._bandits[0].estimator + + def learn_one(self, x, y): + self._n += 1 + self._counter += 1 + + if self.converged: + self._learn_converged(x, y) + else: + self._learn_not_converged(x, y) + + return self + + @property + def best(self): + if not self._converged: + # Lazy selection of the best model + self._sort_bandits() + return self._bandits[0].estimator + + return self._best_estimator + + @property + def converged(self): + return self._converged + + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) + + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) + + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) + + def debug_one(self, x, **kwargs): + try: + return self.best.debug_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) diff --git a/river_extra/model_selection/hyper_band.py b/river_extra/model_selection/hyper_band.py new file mode 100644 index 0000000..4286a30 --- /dev/null +++ b/river_extra/model_selection/hyper_band.py @@ -0,0 +1,437 @@ +import collections +import copy +import numbers +import random +import typing + +import numpy as np +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics +from scipy.stats import qmc + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + +class Hyper_Band(base.Estimator): + """Bandit Self Parameter Tuning + + Parameters + ---------- + estimator + metric + params_range + drift_input + grace_period + drift_detector + convergence_sphere + seed + + References + ---------- + + """ + + _START_RANDOM = "random" + _START_WARM = "warm" + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + nr_estimators: int = 50, + seed: int = None, + ): + super().__init__() + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + + self.nr_estimators = nr_estimators + + self.seed = seed + + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + + sampler = qmc.LatinHypercube(d=len(params_range[self.estimator[1].__class__.__name__])) + self._sample = sampler.random(n=nr_estimators) + self._i=0 + self._p=0 + self._counter=0 + self._best_estimator = None + self._bandits = self._create_bandits(estimator,nr_estimators) + + # Convergence criterion + self._old_centroid = None + + self._pruned_configurations=[] + # Meta-programming + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" + + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return int (val) + elif hp_type == float: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return val + + def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ + + # Sub-component needs to be instantiated + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 + + if operation == "combine": + est_2 = est_2[1] + else: + est_2 = {} + + sub_config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name + else: + sub_hp_prefix = None + + sub_config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], + func=func, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(est_1, numbers.Number): + if operation == "generate": + return self.__generate(hp_data) + if operation == "scale": + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) + return + # combine + return self.__combine(hp_data, est_1, est_2, func) + + # The sub-parameters need to be expanded + config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] + + if operation == "combine": + sub_est_2 = est_2[sub_hp_name] + else: + sub_est_2 = {} + + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name + else: + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + + return config + + def _random_config(self): + return self._traverse_hps( + operation="generate", + hp_data=self.params_range, + est_1=self.estimator._get_params(), + ) + + def _create_bandits(self, model, nr_estimators) -> typing.List: + bandits = [None] * nr_estimators + + for i in range(nr_estimators): + bandits[i] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True), + ) + self._i+=1 + self._p=0 + self._i=0 + return bandits + + def _sort_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._bandits.sort(key=lambda mw: mw.metric.get()) + + def _prune_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + self._bandits[0].estimator._get_params() + half = int(len(self._bandits) /2) + if len(self._bandits)-half>1: + self._bandits=self._bandits[:-half] + else: + self._bandits=self._bandits[:-(len(self._bandits)-1)] + + + def _gen_new_estimator(self, e1, e2, func): + """Generate new configuration given two estimators and a combination function.""" + + est_1_hps = e1.estimator._get_params() + est_2_hps = e2.estimator._get_params() + + new_config = self._traverse_hps( + operation="combine", + hp_data=self.params_range, + est_1=est_1_hps, + func=func, + est_2=est_2_hps + ) + + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(self._bandits[0].estimator), + self.metric.clone(include_attributes=True), + ) + new.estimator.mutate(new_config) + + return new + + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._traverse_hps( + operation="scale", + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled + ) + return scaled + + @property + def _models_converged(self) -> bool: + if len(self._bandits)==3: + return True + else: + return False + + def _learn_converged(self, x, y): + scorer = getattr(self._best_estimator, self._scorer_name) + y_pred = scorer(x) + + input = self.drift_input(y, y_pred) + self.drift_detector.update(input) + + # We need to start the optimization process from scratch + if self.drift_detector.drift_detected: + self._n = 0 + self._converged = False + self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) + + # There is no proven best model right now + self._best_estimator = None + return + + self._best_estimator.learn_one(x, y) + + def _learn_not_converged(self, x, y): + for wrap in self._bandits: + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) + wrap.metric.update(y, y_pred) + wrap.estimator.learn_one(x, y) + + # Keep the simplex ordered + self._sort_bandits() + + if self._n == self.grace_period: + self._n = 0 + self._prune_bandits() + df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] + df3 = df3.reset_index() + importance=self.analyze_fanova(df3) + print(importance) + #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] + + #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] + print("Nr bandits: ",len(self._bandits)) + # 1. Simplex in sphere + scaled_params_b = self._normalize_flattened_hyperspace( + self._bandits[0].estimator._get_params(), + ) + scaled_params_g = self._normalize_flattened_hyperspace( + self._bandits[1].estimator._get_params(), + ) + scaled_params_w = self._normalize_flattened_hyperspace( + self._bandits[2].estimator._get_params(), + ) + #for i in range(1,len(self._bandits)): + # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) + + print("----------") + print( + "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric + ) + print( + "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric + ) + print( + "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric + ) + hyper_points = [ + list(scaled_params_b.values()), + list(scaled_params_g.values()), + list(scaled_params_w.values()), + ] + vectors = np.array(hyper_points) + self._old_centroid = dict( + zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) + ) + + + if self._models_converged: + self._converged = True + self._best_estimator = self._bandits[0].estimator + + def learn_one(self, x, y): + self._n += 1 + self._counter += 1 + + if self.converged: + self._learn_converged(x, y) + else: + self._learn_not_converged(x, y) + + return self + + @property + def best(self): + if not self._converged: + # Lazy selection of the best model + self._sort_bandits() + return self._bandits[0].estimator + + return self._best_estimator + + @property + def converged(self): + return self._converged + + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) + + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) + + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) + + def debug_one(self, x, **kwargs): + try: + return self.best.debug_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) diff --git a/river_extra/model_selection/random_search.py b/river_extra/model_selection/random_search.py new file mode 100644 index 0000000..ba95ec1 --- /dev/null +++ b/river_extra/model_selection/random_search.py @@ -0,0 +1,555 @@ +import statistics +from ast import operator +import collections +import copy +import math +import numbers +import random +import typing +import pandas as pd +import numpy as np +from itertools import combinations +from tqdm import tqdm + +import numpy as np + +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + +class Random_Search(base.Estimator): + """Bandit Self Parameter Tuning + + Parameters + ---------- + estimator + metric + params_range + drift_input + grace_period + drift_detector + convergence_sphere + seed + + References + ---------- + [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning + for data streams. Information Fusion, 76, 75-86. + + """ + + _START_RANDOM = "random" + _START_WARM = "warm" + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + nr_estimators: int = 50, + seed: int = None, + ): + super().__init__() + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + + self.nr_estimators = nr_estimators + + self.seed = seed + + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + grids = np.meshgrid(np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__]))) + self._sample=np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) + + self._df_hpi = pd.DataFrame(columns=['params', 'instances', 'score']) + + sampler = qmc.LatinHypercube(d=len(params_range[self.estimator[1].__class__.__name__])) + self._sample = sampler.random(n=nr_estimators) + self._i=0 + self._p=0 + self._counter=0 + self._best_estimator = None + self._bandits = self._create_bandits(estimator,nr_estimators) + + # Convergence criterion + self._old_centroid = None + + self._pruned_configurations=[] + # Meta-programming + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" + + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return int (val) + elif hp_type == float: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return val + + def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def __combine_bee(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_est_1>=hp_est_2: + func=lambda h1, h2: h2 + ((h1 + h2)*9 / 10) + else: + func = lambda h1, h2: h1 + ((h1 + h2)*9 / 10) + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def analyze_fanova(self, cal_df, max_iter=-1): + metric='score' + params = list(eval(cal_df.loc[0, 'params']).keys()) + result = pd.DataFrame(cal_df.loc[:, metric].copy()) + tmp_df = cal_df.loc[:, 'params'].copy() + for key in params: + result.insert(loc=0, column=key, value=tmp_df.apply(lambda x: eval(x)[key])) + col_name = params[:] + col_name.append(metric) + cal_df = result.reindex(columns=col_name).copy() + + axis = ['id'] + axis.extend(list(cal_df.columns)) + params = axis[1:-1] + metric = axis[-1] + f = pd.DataFrame(columns=axis) + f.loc[0, :] = np.nan + f.loc[0, metric] = cal_df[metric].mean() + f.loc[0, 'id'] = hash(str([])) + v_all = np.std(cal_df[metric].to_numpy()) ** 2 + v = pd.DataFrame(columns=['u', 'v_u', 'F_u(v_u/v_all)']) + for k in range(1, len(params) + 1): + for u in combinations(params, k): + ne_u = set(params) - set(u) + a_u = cal_df.groupby(list(u)).mean().reset_index() + if len(ne_u): + for nu in ne_u: + a_u.loc[:, nu] = np.nan + col_name = cal_df.columns.tolist() + a_u = a_u.reindex(columns=col_name) + sum_f_w = pd.DataFrame(columns=f.columns[:]) + tmp = [] + w_list = [] + for i in range(len(u)): + tmp.extend(list(combinations(u, i))) + for t in tmp: + w_list.append(list(t)) + + for w in w_list: + sum_f_w = pd.concat([sum_f_w, f[f['id'] == hash(str(w))]], ignore_index=True) + col_name = sum_f_w.columns.tolist() + a_u = a_u.reindex(columns=col_name) + for row_index, r in sum_f_w.iterrows(): + r2 = r[1:-1] + not_null_index = r2.notnull().values + if not not_null_index.any(): + a_u.loc[:, col_name[-1]] -= r[-1] + else: + left, right = a_u.iloc[:, 1:-1].align(r2[not_null_index], axis=1, copy=False) + equal_index = (left == right).values.sum(axis=1) + a_u.loc[equal_index == not_null_index.sum(), col_name[-1]] -= r[-1] + a_u['id'] = hash(str(list(u))) + f = pd.concat([f, a_u], ignore_index=True) + tmp_f_u = a_u.loc[:, metric].to_numpy() + v = pd.concat([v, pd.DataFrame( + [{'u': u, 'v_u': (tmp_f_u ** 2).mean(), 'F_u(v_u/v_all)': (tmp_f_u ** 2).mean() / v_all}])], + ignore_index=True) + + return v + + + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ + + # Sub-component needs to be instantiated + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 + + if operation == "combine": + est_2 = est_2[1] + else: + est_2 = {} + + sub_config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name + else: + sub_hp_prefix = None + + sub_config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], + func=func, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(est_1, numbers.Number): + if operation == "generate": + return self.__generate(hp_data) + if operation == "scale": + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) + return + # combine + return self.__combine(hp_data, est_1, est_2, func) + + # The sub-parameters need to be expanded + config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] + + if operation == "combine": + sub_est_2 = est_2[sub_hp_name] + else: + sub_est_2 = {} + + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name + else: + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + + return config + + def _random_config(self): + return self._traverse_hps( + operation="generate", + hp_data=self.params_range, + est_1=self.estimator._get_params(), + ) + + def _create_bandits(self, model, nr_estimators) -> typing.List: + bandits = [None] * nr_estimators + + for i in range(nr_estimators): + bandits[i] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True), + ) + self._i+=1 + self._p=0 + self._i=0 + return bandits + + def _sort_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._bandits.sort(key=lambda mw: mw.metric.get()) + + def _prune_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + self._bandits[0].estimator._get_params() + half = int(1*len(self._bandits) / 10) + for i in range(len(self._bandits)): + #if i>half: + lst=list(self.params_range[self.estimator[1].__class__.__name__].keys()) + mydict={} + for x in lst: + mydict[x]=self._bandits[i].estimator._get_params()[self.estimator[1].__class__.__name__][x] + #mydict['instances'] + self._df_hpi.loc[len(self._df_hpi.index)] = [str(mydict), self._bandits[i].estimator[self.estimator[1].__class__.__name__].summary['total_observed_weight'], self._bandits[i].metric.get()] + #'{\'hp1\': ' + str(row[1]['hp1']) + ', \'hp2\': ' + str(row[1]['hp2']) + ', \'hp3\': ' + str(row[1]['hp3']) + '}' + #new_col = [] + #for row in df.iterrows(): + # new_col.append( + # ) + #df['params'] = new_col + self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ + str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) + #else: + #self._pruned_configurations.append( + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ + # 'grace_period']) + ',' + + # str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ + # 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) + + #if len(self._bandits)-half>2: + # self._bandits=self._bandits[:-half] + #else: + # self._bandits=self._bandits[:-(len(self._bandits)-3)] + + + def _gen_new_estimator(self, e1, e2, func): + """Generate new configuration given two estimators and a combination function.""" + + est_1_hps = e1.estimator._get_params() + est_2_hps = e2.estimator._get_params() + + new_config = self._traverse_hps( + operation="combine", + hp_data=self.params_range, + est_1=est_1_hps, + func=func, + est_2=est_2_hps + ) + + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(self._bandits[0].estimator), + self.metric.clone(include_attributes=True), + ) + new.estimator.mutate(new_config) + + return new + + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._traverse_hps( + operation="scale", + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled + ) + return scaled + + @property + def _models_converged(self) -> bool: + if len(self._bandits)==3: + return True + else: + return False + + def _learn_converged(self, x, y): + scorer = getattr(self._best_estimator, self._scorer_name) + y_pred = scorer(x) + + input = self.drift_input(y, y_pred) + self.drift_detector.update(input) + + # We need to start the optimization process from scratch + if self.drift_detector.drift_detected: + self._n = 0 + self._converged = False + self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) + + # There is no proven best model right now + self._best_estimator = None + return + + self._best_estimator.learn_one(x, y) + + def _learn_not_converged(self, x, y): + for wrap in self._bandits: + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) + wrap.metric.update(y, y_pred) + wrap.estimator.learn_one(x, y) + + # Keep the simplex ordered + self._sort_bandits() + + if self._n == self.grace_period: + self._n = 0 + self._prune_bandits() + df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] + df3 = df3.reset_index() + importance=self.analyze_fanova(df3) + print(importance) + #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] + + #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] + print("Nr bandits: ",len(self._bandits)) + # 1. Simplex in sphere + scaled_params_b = self._normalize_flattened_hyperspace( + self._bandits[0].estimator._get_params(), + ) + scaled_params_g = self._normalize_flattened_hyperspace( + self._bandits[1].estimator._get_params(), + ) + scaled_params_w = self._normalize_flattened_hyperspace( + self._bandits[2].estimator._get_params(), + ) + #for i in range(1,len(self._bandits)): + # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) + + print("----------") + print( + "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric + ) + print( + "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric + ) + print( + "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric + ) + hyper_points = [ + list(scaled_params_b.values()), + list(scaled_params_g.values()), + list(scaled_params_w.values()), + ] + vectors = np.array(hyper_points) + self._old_centroid = dict( + zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) + ) + + + if self._models_converged: + self._converged = True + self._best_estimator = self._bandits[0].estimator + + def learn_one(self, x, y): + self._n += 1 + self._counter += 1 + + if self.converged: + self._learn_converged(x, y) + else: + self._learn_not_converged(x, y) + + return self + + @property + def best(self): + if not self._converged: + # Lazy selection of the best model + self._sort_bandits() + return self._bandits[0].estimator + + return self._best_estimator + + @property + def converged(self): + return self._converged + + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) + + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) + + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) + + def debug_one(self, x, **kwargs): + try: + return self.best.debug_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) From 4d210f5dbf38df85fa1d6efb2c17f1ecc7a088cf Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Thu, 11 May 2023 16:35:42 +0100 Subject: [PATCH 44/50] V1.2.1 fix bugs clean code --- .../model_selection/functional_anova.py | 17 ---- river_extra/model_selection/grid_search.py | 78 ------------------ river_extra/model_selection/random_search.py | 80 +------------------ 3 files changed, 2 insertions(+), 173 deletions(-) diff --git a/river_extra/model_selection/functional_anova.py b/river_extra/model_selection/functional_anova.py index ada5c18..c5b7425 100644 --- a/river_extra/model_selection/functional_anova.py +++ b/river_extra/model_selection/functional_anova.py @@ -123,23 +123,6 @@ def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: new_val = round(new_val, 0) if hp_type == int else new_val return new_val - def __combine_bee(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_est_1>=hp_est_2: - func=lambda h1, h2: h2 + ((h1 + h2)*9 / 10) - else: - func = lambda h1, h2: h1 + ((h1 + h2)*9 / 10) - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - def analyze_fanova(self, cal_df, max_iter=-1): metric='score' params = list(eval(cal_df.loc[0, 'params']).keys()) diff --git a/river_extra/model_selection/grid_search.py b/river_extra/model_selection/grid_search.py index 64a69b2..7e8e84e 100644 --- a/river_extra/model_selection/grid_search.py +++ b/river_extra/model_selection/grid_search.py @@ -120,84 +120,6 @@ def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: new_val = round(new_val, 0) if hp_type == int else new_val return new_val - def __combine_bee(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_est_1>=hp_est_2: - func=lambda h1, h2: h2 + ((h1 + h2)*9 / 10) - else: - func = lambda h1, h2: h1 + ((h1 + h2)*9 / 10) - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - - def analyze_fanova(self, cal_df, max_iter=-1): - metric='score' - params = list(eval(cal_df.loc[0, 'params']).keys()) - result = pd.DataFrame(cal_df.loc[:, metric].copy()) - tmp_df = cal_df.loc[:, 'params'].copy() - for key in params: - result.insert(loc=0, column=key, value=tmp_df.apply(lambda x: eval(x)[key])) - col_name = params[:] - col_name.append(metric) - cal_df = result.reindex(columns=col_name).copy() - - axis = ['id'] - axis.extend(list(cal_df.columns)) - params = axis[1:-1] - metric = axis[-1] - f = pd.DataFrame(columns=axis) - f.loc[0, :] = np.nan - f.loc[0, metric] = cal_df[metric].mean() - f.loc[0, 'id'] = hash(str([])) - v_all = np.std(cal_df[metric].to_numpy()) ** 2 - v = pd.DataFrame(columns=['u', 'v_u', 'F_u(v_u/v_all)']) - for k in range(1, len(params) + 1): - for u in combinations(params, k): - ne_u = set(params) - set(u) - a_u = cal_df.groupby(list(u)).mean().reset_index() - if len(ne_u): - for nu in ne_u: - a_u.loc[:, nu] = np.nan - col_name = cal_df.columns.tolist() - a_u = a_u.reindex(columns=col_name) - sum_f_w = pd.DataFrame(columns=f.columns[:]) - tmp = [] - w_list = [] - for i in range(len(u)): - tmp.extend(list(combinations(u, i))) - for t in tmp: - w_list.append(list(t)) - - for w in w_list: - sum_f_w = pd.concat([sum_f_w, f[f['id'] == hash(str(w))]], ignore_index=True) - col_name = sum_f_w.columns.tolist() - a_u = a_u.reindex(columns=col_name) - for row_index, r in sum_f_w.iterrows(): - r2 = r[1:-1] - not_null_index = r2.notnull().values - if not not_null_index.any(): - a_u.loc[:, col_name[-1]] -= r[-1] - else: - left, right = a_u.iloc[:, 1:-1].align(r2[not_null_index], axis=1, copy=False) - equal_index = (left == right).values.sum(axis=1) - a_u.loc[equal_index == not_null_index.sum(), col_name[-1]] -= r[-1] - a_u['id'] = hash(str(list(u))) - f = pd.concat([f, a_u], ignore_index=True) - tmp_f_u = a_u.loc[:, metric].to_numpy() - v = pd.concat([v, pd.DataFrame( - [{'u': u, 'v_u': (tmp_f_u ** 2).mean(), 'F_u(v_u/v_all)': (tmp_f_u ** 2).mean() / v_all}])], - ignore_index=True) - - return v - - def __flatten(self, prefix, scaled_hps, hp_data, est_data): hp_range = hp_data[1] interval = hp_range[1] - hp_range[0] diff --git a/river_extra/model_selection/random_search.py b/river_extra/model_selection/random_search.py index ba95ec1..113c985 100644 --- a/river_extra/model_selection/random_search.py +++ b/river_extra/model_selection/random_search.py @@ -9,6 +9,8 @@ import pandas as pd import numpy as np from itertools import combinations + +from scipy.stats import qmc from tqdm import tqdm import numpy as np @@ -125,84 +127,6 @@ def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: new_val = round(new_val, 0) if hp_type == int else new_val return new_val - def __combine_bee(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_est_1>=hp_est_2: - func=lambda h1, h2: h2 + ((h1 + h2)*9 / 10) - else: - func = lambda h1, h2: h1 + ((h1 + h2)*9 / 10) - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - - def analyze_fanova(self, cal_df, max_iter=-1): - metric='score' - params = list(eval(cal_df.loc[0, 'params']).keys()) - result = pd.DataFrame(cal_df.loc[:, metric].copy()) - tmp_df = cal_df.loc[:, 'params'].copy() - for key in params: - result.insert(loc=0, column=key, value=tmp_df.apply(lambda x: eval(x)[key])) - col_name = params[:] - col_name.append(metric) - cal_df = result.reindex(columns=col_name).copy() - - axis = ['id'] - axis.extend(list(cal_df.columns)) - params = axis[1:-1] - metric = axis[-1] - f = pd.DataFrame(columns=axis) - f.loc[0, :] = np.nan - f.loc[0, metric] = cal_df[metric].mean() - f.loc[0, 'id'] = hash(str([])) - v_all = np.std(cal_df[metric].to_numpy()) ** 2 - v = pd.DataFrame(columns=['u', 'v_u', 'F_u(v_u/v_all)']) - for k in range(1, len(params) + 1): - for u in combinations(params, k): - ne_u = set(params) - set(u) - a_u = cal_df.groupby(list(u)).mean().reset_index() - if len(ne_u): - for nu in ne_u: - a_u.loc[:, nu] = np.nan - col_name = cal_df.columns.tolist() - a_u = a_u.reindex(columns=col_name) - sum_f_w = pd.DataFrame(columns=f.columns[:]) - tmp = [] - w_list = [] - for i in range(len(u)): - tmp.extend(list(combinations(u, i))) - for t in tmp: - w_list.append(list(t)) - - for w in w_list: - sum_f_w = pd.concat([sum_f_w, f[f['id'] == hash(str(w))]], ignore_index=True) - col_name = sum_f_w.columns.tolist() - a_u = a_u.reindex(columns=col_name) - for row_index, r in sum_f_w.iterrows(): - r2 = r[1:-1] - not_null_index = r2.notnull().values - if not not_null_index.any(): - a_u.loc[:, col_name[-1]] -= r[-1] - else: - left, right = a_u.iloc[:, 1:-1].align(r2[not_null_index], axis=1, copy=False) - equal_index = (left == right).values.sum(axis=1) - a_u.loc[equal_index == not_null_index.sum(), col_name[-1]] -= r[-1] - a_u['id'] = hash(str(list(u))) - f = pd.concat([f, a_u], ignore_index=True) - tmp_f_u = a_u.loc[:, metric].to_numpy() - v = pd.concat([v, pd.DataFrame( - [{'u': u, 'v_u': (tmp_f_u ** 2).mean(), 'F_u(v_u/v_all)': (tmp_f_u ** 2).mean() / v_all}])], - ignore_index=True) - - return v - - def __flatten(self, prefix, scaled_hps, hp_data, est_data): hp_range = hp_data[1] interval = hp_range[1] - hp_range[0] From 427005c89aaddbe289f78d01dec1c9eacb8fd065 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Wed, 8 Nov 2023 13:43:35 +0000 Subject: [PATCH 45/50] V1.2.2 fix bugs clean code --- .gitignore | 1 + river_extra/model_selection/sspt.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index b65cd85..78aac7f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ poetry.lock *.pyc /*.ipynb .DS_Store +/river_extra/model_selection/bandit_g.py diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index c767bc6..1654b1f 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -1,7 +1,3 @@ -<<<<<<< HEAD -from ast import operator -======= ->>>>>>> origin/sspt import collections import copy import math From bc5cb4befe776963aea4868be27579b45f7ef3bc Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Wed, 8 Nov 2023 13:44:47 +0000 Subject: [PATCH 46/50] V1.2.2 fix bugs clean code --- river_extra/model_selection/bandit_d.py | 448 ++++++++++++++++++ .../benchmark_classification.py | 145 ++++++ .../model_selection/classification_test.py | 193 ++++++++ river_extra/model_selection/importance.csv | 4 + .../iris[GridSearchCV]Model1.csv | 26 + .../model_selection/regression_test.py | 79 +++ river_extra/model_selection/teste.ipynb | 57 +++ river_extra/model_selection/teste.py | 25 + 8 files changed, 977 insertions(+) create mode 100644 river_extra/model_selection/bandit_d.py create mode 100644 river_extra/model_selection/benchmark_classification.py create mode 100644 river_extra/model_selection/classification_test.py create mode 100644 river_extra/model_selection/importance.csv create mode 100644 river_extra/model_selection/iris[GridSearchCV]Model1.csv create mode 100644 river_extra/model_selection/regression_test.py create mode 100644 river_extra/model_selection/teste.ipynb create mode 100644 river_extra/model_selection/teste.py diff --git a/river_extra/model_selection/bandit_d.py b/river_extra/model_selection/bandit_d.py new file mode 100644 index 0000000..8fd3d0d --- /dev/null +++ b/river_extra/model_selection/bandit_d.py @@ -0,0 +1,448 @@ +from ast import operator +import collections +import copy +import math +import numbers +import random +import typing + +import numpy as np + +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + +class Bandit(base.Estimator): + """Bandit Self Parameter Tuning + + Parameters + ---------- + estimator + metric + params_range + drift_input + grace_period + drift_detector + convergence_sphere + seed + + References + ---------- + [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning + for data streams. Information Fusion, 76, 75-86. + + """ + + _START_RANDOM = "random" + _START_WARM = "warm" + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + nr_estimators: int = 50, + seed: int = None, + ): + super().__init__() + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + + self.nr_estimators = nr_estimators + + self.seed = seed + + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + from scipy.stats import qmc + sampler = qmc.LatinHypercube(d=3) + self._sample = sampler.random(n=nr_estimators) + + self._i=0 + self._p=0 + + self._best_estimator = None + self._bandits = self._create_bandits(estimator,nr_estimators) + + # Convergence criterion + self._old_centroid = None + + self._pruned_configurations=[] + # Meta-programming + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" + + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return int (val) + elif hp_type == float: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return val + + def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ + + # Sub-component needs to be instantiated + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 + + if operation == "combine": + est_2 = est_2[1] + else: + est_2 = {} + + sub_config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name + else: + sub_hp_prefix = None + + sub_config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], + func=func, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(est_1, numbers.Number): + if operation == "generate": + return self.__generate(hp_data) + if operation == "scale": + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) + return + # combine + return self.__combine(hp_data, est_1, est_2, func) + + # The sub-parameters need to be expanded + config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] + + if operation == "combine": + sub_est_2 = est_2[sub_hp_name] + else: + sub_est_2 = {} + + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name + else: + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + + return config + + def _random_config(self): + return self._traverse_hps( + operation="generate", + hp_data=self.params_range, + est_1=self.estimator._get_params(), + ) + + def _create_bandits(self, model, nr_estimators) -> typing.List: + bandits = [None] * nr_estimators + + for i in range(nr_estimators): + bandits[i] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True), + ) + self._i+=1 + self._p=0 + self._i=0 + return bandits + + def _sort_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._bandits.sort(key=lambda mw: mw.metric.get()) + + def _prune_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + self._bandits[0].estimator._get_params() + half = int(len(self._bandits) / 2) + for i in range(len(self._bandits)): + if i>half: + + + self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ + str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) + else: + self._pruned_configurations.append( + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ + 'grace_period']) + ',' + + str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ + 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) + + #if len(self._bandits)-half>2: + # self._bandits=self._bandits[:-half] + #else: + # self._bandits=self._bandits[:-(len(self._bandits)-3)] + + + def _gen_new_estimator(self, e1, e2, func): + """Generate new configuration given two estimators and a combination function.""" + + est_1_hps = e1.estimator._get_params() + est_2_hps = e2.estimator._get_params() + + new_config = self._traverse_hps( + operation="combine", + hp_data=self.params_range, + est_1=est_1_hps, + func=func, + est_2=est_2_hps + ) + + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(self._bandits[0].estimator), + self.metric.clone(include_attributes=True), + ) + new.estimator.mutate(new_config) + + return new + + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._traverse_hps( + operation="scale", + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled + ) + return scaled + + @property + def _models_converged(self) -> bool: + if len(self._bandits)==3: + return True + else: + return False + + def _learn_converged(self, x, y): + scorer = getattr(self._best_estimator, self._scorer_name) + y_pred = scorer(x) + + input = self.drift_input(y, y_pred) + self.drift_detector.update(input) + + # We need to start the optimization process from scratch + if self.drift_detector.drift_detected: + self._n = 0 + self._converged = False + self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) + + # There is no proven best model right now + self._best_estimator = None + return + + self._best_estimator.learn_one(x, y) + + def _learn_not_converged(self, x, y): + for wrap in self._bandits: + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) + wrap.metric.update(y, y_pred) + wrap.estimator.learn_one(x, y) + + # Keep the simplex ordered + self._sort_bandits() + + if self._n == self.grace_period: + self._n = 0 + self._prune_bandits() + print("Nr bandits: ",len(self._bandits)) + # 1. Simplex in sphere + scaled_params_b = self._normalize_flattened_hyperspace( + self._bandits[0].estimator._get_params(), + ) + scaled_params_g = self._normalize_flattened_hyperspace( + self._bandits[1].estimator._get_params(), + ) + scaled_params_w = self._normalize_flattened_hyperspace( + self._bandits[2].estimator._get_params(), + ) + print("----------") + print( + "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric + ) + print( + "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric + ) + print( + "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric + ) + hyper_points = [ + list(scaled_params_b.values()), + list(scaled_params_g.values()), + list(scaled_params_w.values()), + ] + vectors = np.array(hyper_points) + self._old_centroid = dict( + zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) + ) + + + if self._models_converged: + self._converged = True + self._best_estimator = self._bandits[0].estimator + + def learn_one(self, x, y): + self._n += 1 + + if self.converged: + self._learn_converged(x, y) + else: + self._learn_not_converged(x, y) + + return self + + @property + def best(self): + if not self._converged: + # Lazy selection of the best model + self._sort_bandits() + return self._bandits[0].estimator + + return self._best_estimator + + @property + def converged(self): + return self._converged + + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) + + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) + + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) + + def debug_one(self, x, **kwargs): + try: + return self.best.debug_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) diff --git a/river_extra/model_selection/benchmark_classification.py b/river_extra/model_selection/benchmark_classification.py new file mode 100644 index 0000000..9657ebc --- /dev/null +++ b/river_extra/model_selection/benchmark_classification.py @@ -0,0 +1,145 @@ +import matplotlib.pyplot as plt +from river import datasets, drift, metrics, preprocessing, tree, utils +from river.datasets import synth + +from river_extra.model_selection.bandit_g import Bandit_ps +from river_extra.model_selection.sspt import SSPT +from river_extra.model_selection.bandit_d import Bandit + +# Dataset +dataset = datasets.synth.ConceptDriftStream( + stream=synth.SEA(seed=42, variant=0), + drift_stream=synth.SEA(seed=40, variant=1), + seed=1, + position=10000, + width=2, +).take(20000) + +from river import datasets +#dataset = synth.Hyperplane(seed=42, n_features=2).take(20000) +dataset = datasets.CreditCard().take(20000) + +# Baseline - model and metric +baseline_metric = metrics.Accuracy() +baseline_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) +baseline_metric_plt = [] +baseline_rolling_metric_plt = [] +baseline = preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier() + +# SSPT - model and metric +sspt_metric = metrics.Accuracy() +sspt_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) +sspt_metric_plt = [] +sspt_rolling_metric_plt = [] +sspt = SSPT( + estimator=preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier(), + metric=sspt_rolling_metric, + grace_period=1000, + params_range={ + #"AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, + "HoeffdingTreeClassifier": { + "tau": (float, (0.01, 0.09)), + "delta": (float, (0.00000001, 0.000001)), + "grace_period": (int, (100, 500)), + }, + }, + drift_detector=drift.ADWIN(), + drift_input=lambda yt, yp: 0 if yt == yp else 1, + convergence_sphere=0.000001, + seed=42, +) + +bandit_metric = metrics.Accuracy() +bandit_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) +bandit_metric_plt = [] +bandit_rolling_metric_plt = [] +bandit = Bandit_ps( + estimator=preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier(), + metric=bandit_rolling_metric, + grace_period=1000, + params_range={ + #"AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, + "HoeffdingTreeClassifier": { + "tau": (float, (0.01, 0.1)), + "delta": (float, (0.00001, 0.001)), + "grace_period": (int, (100, 1000)), + "fake_hp": (float, (0.01, 0.1)), + }, + }, + drift_detector=drift.ADWIN(), + drift_input=lambda yt, yp: 0 if yt == yp else 1, + convergence_sphere=0.000001, + nr_estimators=64, + seed=42, +) + +sspt_first_print = True +bandit_first_print = True + +for i, (x, y) in enumerate(dataset): + if i%1000==0: + print(i) + baseline_y_pred = baseline.predict_one(x) + baseline_metric.update(y, baseline_y_pred) + baseline_rolling_metric.update(y, baseline_y_pred) + baseline_metric_plt.append(baseline_metric.get()) + baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) + baseline.learn_one(x, y) + #print('Baseline-------------') + #print(baseline2.debug_one(x)) + + + + + sspt_y_pred = sspt.predict_one(x) + sspt_metric.update(y, sspt_y_pred) + sspt_rolling_metric.update(y, sspt_y_pred) + sspt_metric_plt.append(sspt_metric.get()) + sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) + sspt.learn_one(x, y) + #print('Bandit-------------') + #print(bandit.debug_one(x)) + bandit_y_pred = bandit.predict_one(x) + bandit_metric.update(y, bandit_y_pred) + bandit_rolling_metric.update(y, bandit_y_pred) + bandit_metric_plt.append(bandit_metric.get()) + bandit_rolling_metric_plt.append(bandit_rolling_metric.get()) + bandit.learn_one(x, y) + if sspt.converged and sspt_first_print: + print("SSPT Converged at:", i) + sspt_first_print = False + if bandit.converged and bandit_first_print: + print("Bandit Converged at:", i) + bandit_first_print = False + if sspt.drift_detector.drift_detected: + print("SSPT Drift Detected at:",i) + if bandit.drift_detector.drift_detected: + print("Bandit Drift Detected at:",i) + + +print("Total instances:", i + 1) +print(repr(baseline)) +print("SSPT Best params:") +print(repr(sspt.best)) +print("Bandit Best params:") +print(repr(bandit.best)) +print("SSPT: ", sspt_metric) +print("Bandits: ", bandit_metric) +print("Baseline: ", baseline_metric) + + + +plt.plot(baseline_metric_plt[:20000], linestyle="dotted") +plt.plot(sspt_metric_plt[:20000]) +plt.show() + +plt.plot(baseline_rolling_metric_plt[0:40000],'k') +plt.plot(bandit_rolling_metric_plt[0:40000], 'r') +plt.plot(sspt_rolling_metric_plt[0:40000], 'b') +plt.show() + +with open('/Users/brunoveloso/Downloads/ensaio11.csv', 'w') as f: + + for i in bandit._pruned_configurations: + f.write(i+'\n') + diff --git a/river_extra/model_selection/classification_test.py b/river_extra/model_selection/classification_test.py new file mode 100644 index 0000000..030687c --- /dev/null +++ b/river_extra/model_selection/classification_test.py @@ -0,0 +1,193 @@ +import math + +import matplotlib.pyplot as plt +import river +from river import datasets, drift, metrics, preprocessing, tree, utils +from river.datasets import synth + +from river_extra.model_selection.bandit_g import Bandit_ps +from river_extra.model_selection.sspt import SSPT +from river_extra.model_selection.bandit_d import Bandit + +def myfunc(val): + # Dataset + dataset = river.datasets.synth.ConceptDriftStream( + stream=synth.SEA(seed=42, variant=0), + drift_stream=synth.SEA(seed=40, variant=1), + seed=1, + position=10000, + width=2, + ).take(20000) + + from river import datasets + #dataset = synth.Hyperplane(seed=42, n_features=2).take(20000) + dataset = datasets.CreditCard().take(20000) + + # Baseline - model and metric + baseline_metric = metrics.Accuracy() + baseline_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) + baseline_metric_plt = [] + baseline_rolling_metric_plt = [] + baseline = preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier() + + # SSPT - model and metric + sspt_metric = metrics.Accuracy() + sspt_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) + sspt_metric_plt = [] + sspt_rolling_metric_plt = [] + sspt = SSPT( + estimator=preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier(), + metric=sspt_rolling_metric, + grace_period=1000, + params_range={ + #"AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, + "HoeffdingTreeClassifier": { + "tau": (float, (0.01, 0.09)), + "delta": (float, (0.00000001, 0.000001)), + "grace_period": (int, (100, 500)), + }, + }, + drift_detector=drift.ADWIN(), + drift_input=lambda yt, yp: 0 if yt == yp else 1, + convergence_sphere=0.000001, + seed=42, + ) + + bandit_metric = metrics.Accuracy() + bandit_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) + bandit_metric_plt = [] + bandit_rolling_metric_plt = [] + bandit = Bandit_ps( + estimator=preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier(), + metric=bandit_rolling_metric, + grace_period=1000, + params_range={ + #"AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, + "HoeffdingTreeClassifier": { + "tau": (float, (0.01, 0.1)), + "delta": (float, (0.00001, 0.001)), + "grace_period": (int, (100, 1000)), + }, + }, + drift_detector=drift.ADWIN(), + drift_input=lambda yt, yp: 0 if yt == yp else 1, + convergence_sphere=0.000001, + nr_estimators=64, + seed=42, + ) + + sspt_first_print = True + bandit_first_print = True + # given parameters by user + we_epsilon = 0.1 + we_delta = 0.1 + we_miu = 0 + we_sig2_sum = 0 + we_dev2_sum = 0 + we_dev2 = 0 + we_sig2 = 0 + dyn_w=[] + dyn_n=[] + sig2=[] + dev2=[] + vari = 0 + for i, (x, y) in enumerate(dataset): + if i%1000==0: + print(i) + baseline_y_pred = baseline.predict_one(x) + baseline_metric.update(y, baseline_y_pred) + baseline_rolling_metric.update(y, baseline_y_pred) + baseline_metric_plt.append(baseline_metric.get()) + baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) + baseline.learn_one(x, y) + #print('Baseline-------------') + #print(baseline2.debug_one(x)) + + + + we_lambda = (math.e - 2) + we_upsilon = 4 * we_lambda * math.log(2 / we_delta) / math.pow(we_epsilon, 2) + old_miu=we_miu + we_miu = we_miu+(baseline_rolling_metric.get()-we_miu)/(i+1) + vari = vari + (baseline_rolling_metric.get() - old_miu) * (baseline_rolling_metric.get() - we_miu) + stdn = math.sqrt(vari / (i + 1)) + + we_sig2_sum+=math.pow(baseline_rolling_metric.get(),2) + we_dev2_sum += baseline_rolling_metric.get() + + we_dev2 = (1/(i+1))*we_sig2_sum - math.pow((1/(i+1)*we_dev2_sum),2) + dev2.append((stdn)) + we_n = math.pow(1.96, 2) * stdn*(1-stdn)/(math.pow(0.01,2)) + dyn_n.append(we_n) + + we_sig2 = we_sig2_sum-(i+1)*math.pow(we_miu,2) + sig2.append(we_sig2) + we_ro = max(we_sig2, we_epsilon * we_miu) + if we_miu>0: + we_n = we_upsilon * we_ro / math.pow(we_miu, 2) + dyn_w.append(we_n) + print(we_n) + + # sspt_y_pred = sspt.predict_one(x) + # sspt_metric.update(y, sspt_y_pred) + # sspt_rolling_metric.update(y, sspt_y_pred) + # sspt_metric_plt.append(sspt_metric.get()) + # sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) + # sspt.learn_one(x, y) + # #print('Bandit-------------') + # #print(bandit.debug_one(x)) + # bandit_y_pred = bandit.predict_one(x) + # bandit_metric.update(y, bandit_y_pred) + # bandit_rolling_metric.update(y, bandit_y_pred) + # bandit_metric_plt.append(bandit_metric.get()) + # bandit_rolling_metric_plt.append(bandit_rolling_metric.get()) + # bandit.learn_one(x, y) + # if sspt.converged and sspt_first_print: + # print("SSPT Converged at:", i) + # sspt_first_print = False + # if bandit.converged and bandit_first_print: + # print("Bandit Converged at:", i) + # bandit_first_print = False + # if sspt.drift_detector.drift_detected: + # print("SSPT Drift Detected at:",i) + # if bandit.drift_detector.drift_detected: + # print("Bandit Drift Detected at:",i) + + + print("Total instances:", i + 1) + print(repr(baseline)) + print("SSPT Best params:") + print(repr(sspt.best)) + print("Bandit Best params:") + print(repr(bandit.best)) + print("SSPT: ", sspt_metric) + print("Bandits: ", bandit_metric) + print("Baseline: ", baseline_metric) + + plt.plot(dyn_w, linestyle="dotted") + #plt.plot(dyn_n) + plt.show() + plt.plot(baseline_rolling_metric_plt[500:]) + plt.show() + + # plt.plot(baseline_metric_plt[:20000], linestyle="dotted") + # plt.plot(sspt_metric_plt[:20000]) + # plt.show() + # + # plt.plot(baseline_rolling_metric_plt[0:40000],'k') + # plt.plot(bandit_rolling_metric_plt[0:40000], 'r') + # plt.plot(sspt_rolling_metric_plt[0:40000], 'b') + # plt.show() + + with open('/Users/brunoveloso/Downloads/ensaio11.csv', 'w') as f: + + for i in bandit._pruned_configurations: + f.write(i+'\n') + + + +from bigO import BigO + +lib=BigO() +complexity=lib.test(myfunc,"random") +print(complexity) \ No newline at end of file diff --git a/river_extra/model_selection/importance.csv b/river_extra/model_selection/importance.csv new file mode 100644 index 0000000..6e50a3d --- /dev/null +++ b/river_extra/model_selection/importance.csv @@ -0,0 +1,4 @@ +,u,v_u,F_u(v_u/v_all) +0,"('alpha',)",0.05688453514739228,0.8920573509330904 +1,"('l1_ratio',)",0.00248888888888889,0.03903049613108786 +2,"('alpha', 'l1_ratio')",0.004394376417233563,0.06891215293582162 diff --git a/river_extra/model_selection/iris[GridSearchCV]Model1.csv b/river_extra/model_selection/iris[GridSearchCV]Model1.csv new file mode 100644 index 0000000..0b7e0b0 --- /dev/null +++ b/river_extra/model_selection/iris[GridSearchCV]Model1.csv @@ -0,0 +1,26 @@ +,mean_fit_time,std_fit_time,mean_score_time,std_score_time,param_alpha,param_l1_ratio,params,split0_test_score,split1_test_score,split2_test_score,mean_test_score,std_test_score,rank_test_score +0,0.0038988590240478516,0.00019350259080068593,0.048513333002726235,0.007620761818638512,0.0009765625,0.0,"{'alpha': 0.0009765625, 'l1_ratio': 0.0}",0.8285714285714286,0.9714285714285714,0.9714285714285714,0.9238095238095237,0.06734350297014735,4 +1,0.0034006436665852866,0.0005841867367244808,0.042453765869140625,0.011295176044315678,0.0009765625,0.25,"{'alpha': 0.0009765625, 'l1_ratio': 0.25}",0.8857142857142857,0.9714285714285714,0.9428571428571428,0.9333333333333332,0.03563483225498993,3 +2,0.002705812454223633,0.0005023765548274845,0.04854400952657064,0.009058915009452781,0.0009765625,0.5,"{'alpha': 0.0009765625, 'l1_ratio': 0.5}",0.8857142857142857,1.0,0.9428571428571428,0.9428571428571427,0.04665694748158436,1 +3,0.0033037662506103516,0.0005313131511568678,0.04070862134297689,0.0030310135946994684,0.0009765625,0.75,"{'alpha': 0.0009765625, 'l1_ratio': 0.75}",0.8857142857142857,0.9142857142857143,0.9142857142857143,0.9047619047619047,0.01346870059402948,5 +4,0.0018006960550944011,0.00011594471970534648,0.0002894401550292969,1.3626756826625067e-05,0.0009765625,1.0,"{'alpha': 0.0009765625, 'l1_ratio': 1.0}",0.8857142857142857,1.0,0.9428571428571428,0.9428571428571427,0.04665694748158436,1 +5,0.002614736557006836,0.0006447643344046145,0.00036334991455078125,6.784366740978792e-05,0.03125,0.0,"{'alpha': 0.03125, 'l1_ratio': 0.0}",0.7428571428571429,0.9428571428571428,0.9142857142857143,0.8666666666666667,0.08832017614757812,6 +6,0.002805948257446289,0.00011343430505195735,0.0003763834635416667,1.5144532979525363e-05,0.03125,0.25,"{'alpha': 0.03125, 'l1_ratio': 0.25}",0.7714285714285715,0.8857142857142857,0.9142857142857143,0.8571428571428571,0.06172133998483673,8 +7,0.0027430057525634766,0.00018951542775052176,0.0003916422526041667,4.8637909564400674e-05,0.03125,0.5,"{'alpha': 0.03125, 'l1_ratio': 0.5}",0.7428571428571429,0.9142857142857143,0.9142857142857143,0.8571428571428571,0.08081220356417683,8 +8,0.0027420520782470703,0.0002221225341207598,0.0003730456034342448,1.4321441516853438e-05,0.03125,0.75,"{'alpha': 0.03125, 'l1_ratio': 0.75}",0.8,0.8285714285714286,0.9142857142857143,0.8476190476190476,0.048562090605645536,10 +9,0.0027184486389160156,0.0001826884858045175,0.00036780039469401043,2.4055740185533338e-05,0.03125,1.0,"{'alpha': 0.03125, 'l1_ratio': 1.0}",0.8285714285714286,0.8857142857142857,0.8857142857142857,0.8666666666666667,0.02693740118805891,6 +10,0.002456823984781901,0.0003652350772010518,0.00036676724751790363,6.140912660445521e-05,1.0,0.0,"{'alpha': 1.0, 'l1_ratio': 0.0}",0.7142857142857143,0.6857142857142857,0.7142857142857143,0.7047619047619048,0.01346870059402948,11 +11,0.002367417017618815,0.0004354789732352187,0.0003585020701090495,7.064605302385832e-05,1.0,0.25,"{'alpha': 1.0, 'l1_ratio': 0.25}",0.6857142857142857,0.6857142857142857,0.7142857142857143,0.6952380952380953,0.01346870059402948,12 +12,0.0028084119160970054,5.444453551458701e-05,0.0003689130147298177,8.18145975216623e-06,1.0,0.5,"{'alpha': 1.0, 'l1_ratio': 0.5}",0.6857142857142857,0.6857142857142857,0.45714285714285713,0.6095238095238095,0.10774960475223583,13 +13,0.0017545223236083984,9.122868150470193e-05,0.00028316179911295575,2.7951182098326422e-05,1.0,0.75,"{'alpha': 1.0, 'l1_ratio': 0.75}",0.3142857142857143,0.37142857142857144,0.34285714285714286,0.3428571428571428,0.02332847374079218,20 +14,0.0028853416442871094,0.00012350005685968156,0.00037741661071777344,7.251871230888679e-05,1.0,1.0,"{'alpha': 1.0, 'l1_ratio': 1.0}",0.3142857142857143,0.3142857142857143,0.37142857142857144,0.3333333333333333,0.02693740118805896,21 +15,0.0020644664764404297,0.0005100051870199706,0.0003059705098470052,3.298822581682996e-05,32.0,0.0,"{'alpha': 32.0, 'l1_ratio': 0.0}",0.37142857142857144,0.3142857142857143,0.7142857142857143,0.4666666666666666,0.17664035229515626,15 +16,0.002279361089070638,0.0003502445116967939,0.00038584073384602863,7.185155265996111e-06,32.0,0.25,"{'alpha': 32.0, 'l1_ratio': 0.25}",0.37142857142857144,0.37142857142857144,0.37142857142857144,0.37142857142857144,0.0,16 +17,0.0024990240732828775,0.0004403886677270806,0.0003437201182047526,4.276422691293242e-05,32.0,0.5,"{'alpha': 32.0, 'l1_ratio': 0.5}",0.3142857142857143,0.37142857142857144,0.37142857142857144,0.3523809523809524,0.026937401188058964,18 +18,0.002029975255330404,0.00037354139523400955,0.0003112951914469401,4.8085086434913315e-05,32.0,0.75,"{'alpha': 32.0, 'l1_ratio': 0.75}",0.3142857142857143,0.37142857142857144,0.37142857142857144,0.3523809523809524,0.026937401188058964,18 +19,0.002227306365966797,0.00039614039127220334,0.0003238519032796224,5.381304453676507e-05,32.0,1.0,"{'alpha': 32.0, 'l1_ratio': 1.0}",0.37142857142857144,0.37142857142857144,0.37142857142857144,0.37142857142857144,0.0,16 +20,0.0023507277170817056,0.0005203862341675505,0.0003177324930826823,3.9434718072432945e-05,1024.0,0.0,"{'alpha': 1024.0, 'l1_ratio': 0.0}",0.7714285714285715,0.37142857142857144,0.34285714285714286,0.49523809523809526,0.19564417699213466,14 +21,0.002264738082885742,0.00041749469149180644,0.0003310839335123698,3.608995478017171e-05,1024.0,0.25,"{'alpha': 1024.0, 'l1_ratio': 0.25}",0.3142857142857143,0.3142857142857143,0.37142857142857144,0.3333333333333333,0.02693740118805896,21 +22,0.0018083254496256511,0.0001286763033791477,0.00031304359436035156,4.640942831202071e-05,1024.0,0.5,"{'alpha': 1024.0, 'l1_ratio': 0.5}",0.3142857142857143,0.37142857142857144,0.2857142857142857,0.3238095238095238,0.03563483225498993,24 +23,0.0018104712168375652,0.00016673333992452716,0.00029699007670084637,5.461479925405132e-05,1024.0,0.75,"{'alpha': 1024.0, 'l1_ratio': 0.75}",0.3142857142857143,0.3142857142857143,0.37142857142857144,0.3333333333333333,0.02693740118805896,21 +24,0.0021660327911376953,0.0001316696688384735,0.00031280517578125,5.5347355293830595e-05,1024.0,1.0,"{'alpha': 1024.0, 'l1_ratio': 1.0}",0.3142857142857143,0.3142857142857143,0.2857142857142857,0.30476190476190473,0.01346870059402948,25 diff --git a/river_extra/model_selection/regression_test.py b/river_extra/model_selection/regression_test.py new file mode 100644 index 0000000..3b22445 --- /dev/null +++ b/river_extra/model_selection/regression_test.py @@ -0,0 +1,79 @@ +import matplotlib.pyplot as plt +from river import datasets, drift, linear_model, metrics, preprocessing, utils, rules + +from river_extra import model_selection + +# Dataset +dataset = datasets.synth.FriedmanDrift( + drift_type="gra", position=(7000, 9000), seed=42 +).take(20000) + + +# Baseline - model and metric +baseline_metric = metrics.RMSE() +baseline_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) +baseline_metric_plt = [] +baseline_rolling_metric_plt = [] +baseline = preprocessing.MinMaxScaler() | rules.AMRules() + +# SSPT - model and metric +sspt_metric = metrics.RMSE() +sspt_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) +sspt_metric_plt = [] +sspt_rolling_metric_plt = [] +sspt = model_selection.SSPT( + estimator=preprocessing.MinMaxScaler() | rules.AMRules(), + metric=metrics.RMSE(), + grace_period=100, + params_range={ + #"MinMaxScaler": {"alpha": (float, (0.25, 0.35))}, + "AMRules": { + "tau": (float, (0.01, 0.1)), + "delta": (float, (0.00001, 0.001)), + "n_min": (int, (50, 500)) + }, + }, + drift_input=lambda yt, yp: abs(yt - yp), + drift_detector=drift.PageHinkley(), + convergence_sphere=0.000001, + seed=42, +) + +first_print = True + +metric = metrics.RMSE() + + +for i, (x, y) in enumerate(dataset): + baseline_y_pred = baseline.predict_one(x) + baseline_metric.update(y, baseline_y_pred) + baseline_rolling_metric.update(y, baseline_y_pred) + baseline_metric_plt.append(baseline_metric.get()) + baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) + baseline.learn_one(x, y) + sspt_y_pred = sspt.predict_one(x) + sspt_metric.update(y, sspt_y_pred) + sspt_rolling_metric.update(y, sspt_y_pred) + sspt_metric_plt.append(sspt_metric.get()) + sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) + sspt.learn_one(x, y) + + if sspt.converged and first_print: + print("Converged at:", i) + first_print = False + +print("Total instances:", i + 1) +print(repr(baseline)) +print("Best params:") +print(repr(sspt.best)) +print("SSPT: ", sspt_metric) +print("Baseline: ", baseline_metric) + + +#plt.plot(baseline_metric_plt[:10000], linestyle="dotted") +#plt.plot(sspt_metric_plt[:10000]) +#plt.show() + +plt.plot(baseline_rolling_metric_plt[:20000], linestyle="dotted") +plt.plot(sspt_rolling_metric_plt[:20000]) +plt.show() diff --git a/river_extra/model_selection/teste.ipynb b/river_extra/model_selection/teste.ipynb new file mode 100644 index 0000000..6a33c37 --- /dev/null +++ b/river_extra/model_selection/teste.ipynb @@ -0,0 +1,57 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2023-05-12T15:38:06.947205Z", + "start_time": "2023-05-12T15:38:06.927809Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "teste\n" + ] + } + ], + "source": [ + "print('teste')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/river_extra/model_selection/teste.py b/river_extra/model_selection/teste.py new file mode 100644 index 0000000..b30fd82 --- /dev/null +++ b/river_extra/model_selection/teste.py @@ -0,0 +1,25 @@ +import hyanova + +import pandas as pd +import numpy as np +from itertools import combinations +from tqdm import tqdm + + + + +df = pd.read_csv('/Users/brunoveloso/Downloads/ensaio11.csv', names=['hp1', 'hp2', 'hp3', 'instances', 'score']) +print(df) + + + +for i in df.instances.unique(): + print('------->' + str(i)) + df3=df[df['instances']==i] + df3=df3.reset_index() + df2, params = hyanova.read_df(df3, 'score') + #print(df2['score'].unique()) + importance = analyze_incr(df2,max_iter=-1) + print(importance) + #break + From 031fc090cec1497c90af0c6c5958850d8351d569 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Wed, 8 Nov 2023 13:45:25 +0000 Subject: [PATCH 47/50] V1.2.2 fix bugs clean code --- river_extra/model_selection/bandit_g.py | 555 ++++++++++++++++++++++++ 1 file changed, 555 insertions(+) create mode 100644 river_extra/model_selection/bandit_g.py diff --git a/river_extra/model_selection/bandit_g.py b/river_extra/model_selection/bandit_g.py new file mode 100644 index 0000000..9998ef7 --- /dev/null +++ b/river_extra/model_selection/bandit_g.py @@ -0,0 +1,555 @@ +import statistics +from ast import operator +import collections +import copy +import math +import numbers +import random +import typing +import pandas as pd +import numpy as np +from itertools import combinations +from tqdm import tqdm + +import numpy as np + +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + +class Bandit_ps(base.Estimator): + """Bandit Self Parameter Tuning + + Parameters + ---------- + estimator + metric + params_range + drift_input + grace_period + drift_detector + convergence_sphere + seed + + References + ---------- + [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning + for data streams. Information Fusion, 76, 75-86. + + """ + + _START_RANDOM = "random" + _START_WARM = "warm" + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + nr_estimators: int = 50, + seed: int = None, + ): + super().__init__() + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + + self.nr_estimators = nr_estimators + + self.seed = seed + + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + grids = np.meshgrid(np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__]))) + self._sample=np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) + + self._df_hpi = pd.DataFrame(columns=['params', 'instances', 'score']) + + self._i=0 + self._p=0 + self._counter=0 + self._best_estimator = None + self._bandits = self._create_bandits(estimator,nr_estimators) + + # Convergence criterion + self._old_centroid = None + + self._pruned_configurations=[] + # Meta-programming + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" + + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return int (val) + elif hp_type == float: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return val + + def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def __combine_bee(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_est_1>=hp_est_2: + func=lambda h1, h2: h2 + ((h1 + h2)*9 / 10) + else: + func = lambda h1, h2: h1 + ((h1 + h2)*9 / 10) + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def analyze_fanova(self, cal_df, max_iter=-1): + metric='score' + params = list(eval(cal_df.loc[0, 'params']).keys()) + result = pd.DataFrame(cal_df.loc[:, metric].copy()) + tmp_df = cal_df.loc[:, 'params'].copy() + for key in params: + result.insert(loc=0, column=key, value=tmp_df.apply(lambda x: eval(x)[key])) + col_name = params[:] + col_name.append(metric) + cal_df = result.reindex(columns=col_name).copy() + + axis = ['id'] + axis.extend(list(cal_df.columns)) + params = axis[1:-1] + metric = axis[-1] + f = pd.DataFrame(columns=axis) + f.loc[0, :] = np.nan + f.loc[0, metric] = cal_df[metric].mean() + f.loc[0, 'id'] = hash(str([])) + v_all = np.std(cal_df[metric].to_numpy()) ** 2 + v = pd.DataFrame(columns=['u', 'v_u', 'F_u(v_u/v_all)']) + for k in range(1, len(params) + 1): + for u in combinations(params, k): + ne_u = set(params) - set(u) + a_u = cal_df.groupby(list(u)).mean().reset_index() + if len(ne_u): + for nu in ne_u: + a_u.loc[:, nu] = np.nan + col_name = cal_df.columns.tolist() + a_u = a_u.reindex(columns=col_name) + sum_f_w = pd.DataFrame(columns=f.columns[:]) + tmp = [] + w_list = [] + for i in range(len(u)): + tmp.extend(list(combinations(u, i))) + for t in tmp: + w_list.append(list(t)) + + for w in w_list: + sum_f_w = pd.concat([sum_f_w, f[f['id'] == hash(str(w))]], ignore_index=True) + col_name = sum_f_w.columns.tolist() + a_u = a_u.reindex(columns=col_name) + for row_index, r in sum_f_w.iterrows(): + r2 = r[1:-1] + not_null_index = r2.notnull().values + if not not_null_index.any(): + a_u.loc[:, col_name[-1]] -= r[-1] + else: + left, right = a_u.iloc[:, 1:-1].align(r2[not_null_index], axis=1, copy=False) + equal_index = (left == right).values.sum(axis=1) + a_u.loc[equal_index == not_null_index.sum(), col_name[-1]] -= r[-1] + a_u['id'] = hash(str(list(u))) + f = pd.concat([f, a_u], ignore_index=True) + tmp_f_u = a_u.loc[:, metric].to_numpy() + v = pd.concat([v, pd.DataFrame( + [{'u': u, 'v_u': (tmp_f_u ** 2).mean(), 'F_u(v_u/v_all)': (tmp_f_u ** 2).mean() / v_all}])], + ignore_index=True) + + return v + + + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ + + # Sub-component needs to be instantiated + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 + + if operation == "combine": + est_2 = est_2[1] + else: + est_2 = {} + + sub_config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name + else: + sub_hp_prefix = None + + sub_config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], + func=func, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(est_1, numbers.Number): + if operation == "generate": + return self.__generate(hp_data) + if operation == "scale": + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) + return + # combine + return self.__combine(hp_data, est_1, est_2, func) + + # The sub-parameters need to be expanded + config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] + + if operation == "combine": + sub_est_2 = est_2[sub_hp_name] + else: + sub_est_2 = {} + + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name + else: + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + + return config + + def _random_config(self): + return self._traverse_hps( + operation="generate", + hp_data=self.params_range, + est_1=self.estimator._get_params(), + ) + + def _create_bandits(self, model, nr_estimators) -> typing.List: + bandits = [None] * nr_estimators + + for i in range(nr_estimators): + bandits[i] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True), + ) + self._i+=1 + self._p=0 + self._i=0 + return bandits + + def _sort_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._bandits.sort(key=lambda mw: mw.metric.get()) + + def _prune_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + self._bandits[0].estimator._get_params() + half = int(1*len(self._bandits) / 10) + for i in range(len(self._bandits)): + #if i>half: + lst=list(self.params_range[self.estimator[1].__class__.__name__].keys()) + mydict={} + for x in lst: + mydict[x]=self._bandits[i].estimator._get_params()[self.estimator[1].__class__.__name__][x] + #mydict['instances'] + self._df_hpi.loc[len(self._df_hpi.index)] = [str(mydict), self._bandits[i].estimator[self.estimator[1].__class__.__name__].summary['total_observed_weight'], self._bandits[i].metric.get()] + #'{\'hp1\': ' + str(row[1]['hp1']) + ', \'hp2\': ' + str(row[1]['hp2']) + ', \'hp3\': ' + str(row[1]['hp3']) + '}' + #new_col = [] + #for row in df.iterrows(): + # new_col.append( + # ) + #df['params'] = new_col + self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ + str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) + #else: + #self._pruned_configurations.append( + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ + # 'grace_period']) + ',' + + # str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ + # 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) + + #if len(self._bandits)-half>2: + # self._bandits=self._bandits[:-half] + #else: + # self._bandits=self._bandits[:-(len(self._bandits)-3)] + + + def _gen_new_estimator(self, e1, e2, func): + """Generate new configuration given two estimators and a combination function.""" + + est_1_hps = e1.estimator._get_params() + est_2_hps = e2.estimator._get_params() + + new_config = self._traverse_hps( + operation="combine", + hp_data=self.params_range, + est_1=est_1_hps, + func=func, + est_2=est_2_hps + ) + + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(self._bandits[0].estimator), + self.metric.clone(include_attributes=True), + ) + new.estimator.mutate(new_config) + + return new + + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._traverse_hps( + operation="scale", + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled + ) + return scaled + + @property + def _models_converged(self) -> bool: + if len(self._bandits)==3: + return True + else: + return False + + def _learn_converged(self, x, y): + scorer = getattr(self._best_estimator, self._scorer_name) + y_pred = scorer(x) + + input = self.drift_input(y, y_pred) + self.drift_detector.update(input) + + # We need to start the optimization process from scratch + if self.drift_detector.drift_detected: + self._n = 0 + self._converged = False + self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) + + # There is no proven best model right now + self._best_estimator = None + return + + self._best_estimator.learn_one(x, y) + + def _learn_not_converged(self, x, y): + for wrap in self._bandits: + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) + wrap.metric.update(y, y_pred) + wrap.estimator.learn_one(x, y) + + # Keep the simplex ordered + self._sort_bandits() + + if self._n == self.grace_period: + self._n = 0 + self._prune_bandits() + df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] + df3 = df3.reset_index() + importance=self.analyze_fanova(df3) + print(importance) + #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] + + #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] + print("Nr bandits: ",len(self._bandits)) + # 1. Simplex in sphere + scaled_params_b = self._normalize_flattened_hyperspace( + self._bandits[0].estimator._get_params(), + ) + scaled_params_g = self._normalize_flattened_hyperspace( + self._bandits[1].estimator._get_params(), + ) + scaled_params_w = self._normalize_flattened_hyperspace( + self._bandits[2].estimator._get_params(), + ) + #for i in range(1,len(self._bandits)): + # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) + + print("----------") + print( + "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric + ) + print( + "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric + ) + print( + "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric + ) + hyper_points = [ + list(scaled_params_b.values()), + list(scaled_params_g.values()), + list(scaled_params_w.values()), + ] + vectors = np.array(hyper_points) + self._old_centroid = dict( + zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) + ) + + + if self._models_converged: + self._converged = True + self._best_estimator = self._bandits[0].estimator + + def learn_one(self, x, y): + + + self._n += 1 + self._counter += 1 + + if self.converged: + self._learn_converged(x, y) + else: + self._learn_not_converged(x, y) + + return self + + @property + def best(self): + if not self._converged: + # Lazy selection of the best model + self._sort_bandits() + return self._bandits[0].estimator + + return self._best_estimator + + @property + def converged(self): + return self._converged + + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) + + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) + + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) + + def debug_one(self, x, **kwargs): + try: + return self.best.debug_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) From e3459ac575dd0caca9903b69259bd9479097740e Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Wed, 8 Nov 2023 13:45:25 +0000 Subject: [PATCH 48/50] V1.2.2 fix bugs clean code --- river_extra/model_selection/bandit_g.py | 555 ++++++++++++++++++++++++ 1 file changed, 555 insertions(+) create mode 100644 river_extra/model_selection/bandit_g.py diff --git a/river_extra/model_selection/bandit_g.py b/river_extra/model_selection/bandit_g.py new file mode 100644 index 0000000..9998ef7 --- /dev/null +++ b/river_extra/model_selection/bandit_g.py @@ -0,0 +1,555 @@ +import statistics +from ast import operator +import collections +import copy +import math +import numbers +import random +import typing +import pandas as pd +import numpy as np +from itertools import combinations +from tqdm import tqdm + +import numpy as np + +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + +class Bandit_ps(base.Estimator): + """Bandit Self Parameter Tuning + + Parameters + ---------- + estimator + metric + params_range + drift_input + grace_period + drift_detector + convergence_sphere + seed + + References + ---------- + [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning + for data streams. Information Fusion, 76, 75-86. + + """ + + _START_RANDOM = "random" + _START_WARM = "warm" + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + nr_estimators: int = 50, + seed: int = None, + ): + super().__init__() + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + + self.nr_estimators = nr_estimators + + self.seed = seed + + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + grids = np.meshgrid(np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__]))) + self._sample=np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) + + self._df_hpi = pd.DataFrame(columns=['params', 'instances', 'score']) + + self._i=0 + self._p=0 + self._counter=0 + self._best_estimator = None + self._bandits = self._create_bandits(estimator,nr_estimators) + + # Convergence criterion + self._old_centroid = None + + self._pruned_configurations=[] + # Meta-programming + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" + + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return int (val) + elif hp_type == float: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return val + + def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def __combine_bee(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_est_1>=hp_est_2: + func=lambda h1, h2: h2 + ((h1 + h2)*9 / 10) + else: + func = lambda h1, h2: h1 + ((h1 + h2)*9 / 10) + new_val = func(hp_est_1, hp_est_2) + + # Range sanity checks + if new_val < hp_range[0]: + new_val = hp_range[0] + if new_val > hp_range[1]: + new_val = hp_range[1] + + new_val = round(new_val, 0) if hp_type == int else new_val + return new_val + + def analyze_fanova(self, cal_df, max_iter=-1): + metric='score' + params = list(eval(cal_df.loc[0, 'params']).keys()) + result = pd.DataFrame(cal_df.loc[:, metric].copy()) + tmp_df = cal_df.loc[:, 'params'].copy() + for key in params: + result.insert(loc=0, column=key, value=tmp_df.apply(lambda x: eval(x)[key])) + col_name = params[:] + col_name.append(metric) + cal_df = result.reindex(columns=col_name).copy() + + axis = ['id'] + axis.extend(list(cal_df.columns)) + params = axis[1:-1] + metric = axis[-1] + f = pd.DataFrame(columns=axis) + f.loc[0, :] = np.nan + f.loc[0, metric] = cal_df[metric].mean() + f.loc[0, 'id'] = hash(str([])) + v_all = np.std(cal_df[metric].to_numpy()) ** 2 + v = pd.DataFrame(columns=['u', 'v_u', 'F_u(v_u/v_all)']) + for k in range(1, len(params) + 1): + for u in combinations(params, k): + ne_u = set(params) - set(u) + a_u = cal_df.groupby(list(u)).mean().reset_index() + if len(ne_u): + for nu in ne_u: + a_u.loc[:, nu] = np.nan + col_name = cal_df.columns.tolist() + a_u = a_u.reindex(columns=col_name) + sum_f_w = pd.DataFrame(columns=f.columns[:]) + tmp = [] + w_list = [] + for i in range(len(u)): + tmp.extend(list(combinations(u, i))) + for t in tmp: + w_list.append(list(t)) + + for w in w_list: + sum_f_w = pd.concat([sum_f_w, f[f['id'] == hash(str(w))]], ignore_index=True) + col_name = sum_f_w.columns.tolist() + a_u = a_u.reindex(columns=col_name) + for row_index, r in sum_f_w.iterrows(): + r2 = r[1:-1] + not_null_index = r2.notnull().values + if not not_null_index.any(): + a_u.loc[:, col_name[-1]] -= r[-1] + else: + left, right = a_u.iloc[:, 1:-1].align(r2[not_null_index], axis=1, copy=False) + equal_index = (left == right).values.sum(axis=1) + a_u.loc[equal_index == not_null_index.sum(), col_name[-1]] -= r[-1] + a_u['id'] = hash(str(list(u))) + f = pd.concat([f, a_u], ignore_index=True) + tmp_f_u = a_u.loc[:, metric].to_numpy() + v = pd.concat([v, pd.DataFrame( + [{'u': u, 'v_u': (tmp_f_u ** 2).mean(), 'F_u(v_u/v_all)': (tmp_f_u ** 2).mean() / v_all}])], + ignore_index=True) + + return v + + + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ + + # Sub-component needs to be instantiated + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 + + if operation == "combine": + est_2 = est_2[1] + else: + est_2 = {} + + sub_config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name + else: + sub_hp_prefix = None + + sub_config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], + func=func, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(est_1, numbers.Number): + if operation == "generate": + return self.__generate(hp_data) + if operation == "scale": + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) + return + # combine + return self.__combine(hp_data, est_1, est_2, func) + + # The sub-parameters need to be expanded + config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] + + if operation == "combine": + sub_est_2 = est_2[sub_hp_name] + else: + sub_est_2 = {} + + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name + else: + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + + return config + + def _random_config(self): + return self._traverse_hps( + operation="generate", + hp_data=self.params_range, + est_1=self.estimator._get_params(), + ) + + def _create_bandits(self, model, nr_estimators) -> typing.List: + bandits = [None] * nr_estimators + + for i in range(nr_estimators): + bandits[i] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True), + ) + self._i+=1 + self._p=0 + self._i=0 + return bandits + + def _sort_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._bandits.sort(key=lambda mw: mw.metric.get()) + + def _prune_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + self._bandits[0].estimator._get_params() + half = int(1*len(self._bandits) / 10) + for i in range(len(self._bandits)): + #if i>half: + lst=list(self.params_range[self.estimator[1].__class__.__name__].keys()) + mydict={} + for x in lst: + mydict[x]=self._bandits[i].estimator._get_params()[self.estimator[1].__class__.__name__][x] + #mydict['instances'] + self._df_hpi.loc[len(self._df_hpi.index)] = [str(mydict), self._bandits[i].estimator[self.estimator[1].__class__.__name__].summary['total_observed_weight'], self._bandits[i].metric.get()] + #'{\'hp1\': ' + str(row[1]['hp1']) + ', \'hp2\': ' + str(row[1]['hp2']) + ', \'hp3\': ' + str(row[1]['hp3']) + '}' + #new_col = [] + #for row in df.iterrows(): + # new_col.append( + # ) + #df['params'] = new_col + self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ + str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ + str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) + #else: + #self._pruned_configurations.append( + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + + # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ + # 'grace_period']) + ',' + + # str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ + # 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) + + #if len(self._bandits)-half>2: + # self._bandits=self._bandits[:-half] + #else: + # self._bandits=self._bandits[:-(len(self._bandits)-3)] + + + def _gen_new_estimator(self, e1, e2, func): + """Generate new configuration given two estimators and a combination function.""" + + est_1_hps = e1.estimator._get_params() + est_2_hps = e2.estimator._get_params() + + new_config = self._traverse_hps( + operation="combine", + hp_data=self.params_range, + est_1=est_1_hps, + func=func, + est_2=est_2_hps + ) + + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(self._bandits[0].estimator), + self.metric.clone(include_attributes=True), + ) + new.estimator.mutate(new_config) + + return new + + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._traverse_hps( + operation="scale", + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled + ) + return scaled + + @property + def _models_converged(self) -> bool: + if len(self._bandits)==3: + return True + else: + return False + + def _learn_converged(self, x, y): + scorer = getattr(self._best_estimator, self._scorer_name) + y_pred = scorer(x) + + input = self.drift_input(y, y_pred) + self.drift_detector.update(input) + + # We need to start the optimization process from scratch + if self.drift_detector.drift_detected: + self._n = 0 + self._converged = False + self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) + + # There is no proven best model right now + self._best_estimator = None + return + + self._best_estimator.learn_one(x, y) + + def _learn_not_converged(self, x, y): + for wrap in self._bandits: + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) + wrap.metric.update(y, y_pred) + wrap.estimator.learn_one(x, y) + + # Keep the simplex ordered + self._sort_bandits() + + if self._n == self.grace_period: + self._n = 0 + self._prune_bandits() + df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] + df3 = df3.reset_index() + importance=self.analyze_fanova(df3) + print(importance) + #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] + + #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] + print("Nr bandits: ",len(self._bandits)) + # 1. Simplex in sphere + scaled_params_b = self._normalize_flattened_hyperspace( + self._bandits[0].estimator._get_params(), + ) + scaled_params_g = self._normalize_flattened_hyperspace( + self._bandits[1].estimator._get_params(), + ) + scaled_params_w = self._normalize_flattened_hyperspace( + self._bandits[2].estimator._get_params(), + ) + #for i in range(1,len(self._bandits)): + # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) + + print("----------") + print( + "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric + ) + print( + "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric + ) + print( + "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric + ) + hyper_points = [ + list(scaled_params_b.values()), + list(scaled_params_g.values()), + list(scaled_params_w.values()), + ] + vectors = np.array(hyper_points) + self._old_centroid = dict( + zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) + ) + + + if self._models_converged: + self._converged = True + self._best_estimator = self._bandits[0].estimator + + def learn_one(self, x, y): + + + self._n += 1 + self._counter += 1 + + if self.converged: + self._learn_converged(x, y) + else: + self._learn_not_converged(x, y) + + return self + + @property + def best(self): + if not self._converged: + # Lazy selection of the best model + self._sort_bandits() + return self._bandits[0].estimator + + return self._best_estimator + + @property + def converged(self): + return self._converged + + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) + + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) + + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) + + def debug_one(self, x, **kwargs): + try: + return self.best.debug_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) From d8cdf7335626b446bbb6d04cb51a814969562b3b Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Wed, 8 Nov 2023 14:25:23 +0000 Subject: [PATCH 49/50] V1.2.3 clean code --- .gitignore | 1 - river_extra/model_selection/bandit_d.py | 448 -------------- river_extra/model_selection/bandit_g.py | 555 ------------------ .../benchmark_classification.py | 145 ----- .../model_selection/classification_test.py | 193 ------ .../model_selection/functional_anova.py | 536 ----------------- river_extra/model_selection/grid_search.py | 472 --------------- river_extra/model_selection/hyper_band.py | 437 -------------- river_extra/model_selection/importance.csv | 4 - .../iris[GridSearchCV]Model1.csv | 26 - river_extra/model_selection/random_search.py | 479 --------------- .../model_selection/regression_test.py | 79 --- river_extra/model_selection/sspt.py | 2 +- river_extra/model_selection/teste.ipynb | 57 -- river_extra/model_selection/teste.py | 25 - 15 files changed, 1 insertion(+), 3458 deletions(-) delete mode 100644 river_extra/model_selection/bandit_d.py delete mode 100644 river_extra/model_selection/bandit_g.py delete mode 100644 river_extra/model_selection/benchmark_classification.py delete mode 100644 river_extra/model_selection/classification_test.py delete mode 100644 river_extra/model_selection/functional_anova.py delete mode 100644 river_extra/model_selection/grid_search.py delete mode 100644 river_extra/model_selection/hyper_band.py delete mode 100644 river_extra/model_selection/importance.csv delete mode 100644 river_extra/model_selection/iris[GridSearchCV]Model1.csv delete mode 100644 river_extra/model_selection/random_search.py delete mode 100644 river_extra/model_selection/regression_test.py delete mode 100644 river_extra/model_selection/teste.ipynb delete mode 100644 river_extra/model_selection/teste.py diff --git a/.gitignore b/.gitignore index 78aac7f..b65cd85 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,3 @@ poetry.lock *.pyc /*.ipynb .DS_Store -/river_extra/model_selection/bandit_g.py diff --git a/river_extra/model_selection/bandit_d.py b/river_extra/model_selection/bandit_d.py deleted file mode 100644 index 8fd3d0d..0000000 --- a/river_extra/model_selection/bandit_d.py +++ /dev/null @@ -1,448 +0,0 @@ -from ast import operator -import collections -import copy -import math -import numbers -import random -import typing - -import numpy as np - -# TODO use lazy imports where needed -from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree - -ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") - - -class Bandit(base.Estimator): - """Bandit Self Parameter Tuning - - Parameters - ---------- - estimator - metric - params_range - drift_input - grace_period - drift_detector - convergence_sphere - seed - - References - ---------- - [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning - for data streams. Information Fusion, 76, 75-86. - - """ - - _START_RANDOM = "random" - _START_WARM = "warm" - - def __init__( - self, - estimator: base.Estimator, - metric: metrics.base.Metric, - params_range: typing.Dict[str, typing.Tuple], - drift_input: typing.Callable[[float, float], float], - grace_period: int = 500, - drift_detector: base.DriftDetector = drift.ADWIN(), - convergence_sphere: float = 0.001, - nr_estimators: int = 50, - seed: int = None, - ): - super().__init__() - self.estimator = estimator - self.metric = metric - self.params_range = params_range - self.drift_input = drift_input - - self.grace_period = grace_period - self.drift_detector = drift_detector - self.convergence_sphere = convergence_sphere - - self.nr_estimators = nr_estimators - - self.seed = seed - - self._n = 0 - self._converged = False - self._rng = random.Random(self.seed) - from scipy.stats import qmc - sampler = qmc.LatinHypercube(d=3) - self._sample = sampler.random(n=nr_estimators) - - self._i=0 - self._p=0 - - self._best_estimator = None - self._bandits = self._create_bandits(estimator,nr_estimators) - - # Convergence criterion - self._old_centroid = None - - self._pruned_configurations=[] - # Meta-programming - border = self.estimator - if isinstance(border, compose.Pipeline): - border = border[-1] - - if isinstance(border, (base.Classifier, base.Regressor)): - self._scorer_name = "predict_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "score_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "classify" - - def __generate(self, hp_data) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_type == int: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return int (val) - elif hp_type == float: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return val - - def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - - def __flatten(self, prefix, scaled_hps, hp_data, est_data): - hp_range = hp_data[1] - interval = hp_range[1] - hp_range[0] - scaled_hps[prefix] = (est_data - hp_range[0]) / interval - - def _traverse_hps( - self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None - ) -> typing.Optional[typing.Union[dict, numbers.Number]]: - """Traverse the hyperparameters of the estimator/pipeline and perform an operation. - - Parameters - ---------- - operation - The operation that is intented to apply over the hyperparameters. Can be either: - "combine" (combine parameters from two pipelines), "scale" (scale a flattened - version of the hyperparameter hierarchy to use in the stopping criteria), or - "generate" (create a new hyperparameter set candidate). - hp_data - The hyperparameter data which was passed by the user. Defines the ranges to - explore for each hyperparameter. - est_1 - The hyperparameter structure of the first pipeline/estimator. Such structure is obtained - via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. - func - A function that is used to combine the values in `est_1` and `est_2`, in case - `operation="combine"`. - est_2 - A second pipeline/estimator which is going to be combined with `est_1`, in case - `operation="combine"`. - hp_prefix - A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter - hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly - to the current position in the hierarchy. Initially it is set to `None`. - scaled_hps - Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. - Set to `None` and defined automatically when `operation="scale"`. - """ - - # Sub-component needs to be instantiated - if isinstance(est_1, tuple): - sub_class, est_1 = est_1 - - if operation == "combine": - est_2 = est_2[1] - else: - est_2 = {} - - sub_config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name - else: - sub_hp_prefix = None - - sub_config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=est_1[sub_hp_name], - func=func, - est_2=est_2.get(sub_hp_name, None), - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - return sub_class(**sub_config) - - # We reached the numeric parameters - if isinstance(est_1, numbers.Number): - if operation == "generate": - return self.__generate(hp_data) - if operation == "scale": - self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) - return - # combine - return self.__combine(hp_data, est_1, est_2, func) - - # The sub-parameters need to be expanded - config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - sub_est_1 = est_1[sub_hp_name] - - if operation == "combine": - sub_est_2 = est_2[sub_hp_name] - else: - sub_est_2 = {} - - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name - else: - sub_hp_prefix = None - - config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=sub_est_1, - func=func, - est_2=sub_est_2, - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - - return config - - def _random_config(self): - return self._traverse_hps( - operation="generate", - hp_data=self.params_range, - est_1=self.estimator._get_params(), - ) - - def _create_bandits(self, model, nr_estimators) -> typing.List: - bandits = [None] * nr_estimators - - for i in range(nr_estimators): - bandits[i] = ModelWrapper( - model.clone(self._random_config(), include_attributes=True), - self.metric.clone(include_attributes=True), - ) - self._i+=1 - self._p=0 - self._i=0 - return bandits - - def _sort_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - if self.metric.bigger_is_better: - self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) - else: - self._bandits.sort(key=lambda mw: mw.metric.get()) - - def _prune_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - self._bandits[0].estimator._get_params() - half = int(len(self._bandits) / 2) - for i in range(len(self._bandits)): - if i>half: - - - self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ - str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) - else: - self._pruned_configurations.append( - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ - 'grace_period']) + ',' + - str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ - 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) - - #if len(self._bandits)-half>2: - # self._bandits=self._bandits[:-half] - #else: - # self._bandits=self._bandits[:-(len(self._bandits)-3)] - - - def _gen_new_estimator(self, e1, e2, func): - """Generate new configuration given two estimators and a combination function.""" - - est_1_hps = e1.estimator._get_params() - est_2_hps = e2.estimator._get_params() - - new_config = self._traverse_hps( - operation="combine", - hp_data=self.params_range, - est_1=est_1_hps, - func=func, - est_2=est_2_hps - ) - - # Modify the current best contender with the new hyperparameter values - new = ModelWrapper( - copy.deepcopy(self._bandits[0].estimator), - self.metric.clone(include_attributes=True), - ) - new.estimator.mutate(new_config) - - return new - - - def _normalize_flattened_hyperspace(self, orig): - scaled = {} - self._traverse_hps( - operation="scale", - hp_data=self.params_range, - est_1=orig, - hp_prefix="", - scaled_hps=scaled - ) - return scaled - - @property - def _models_converged(self) -> bool: - if len(self._bandits)==3: - return True - else: - return False - - def _learn_converged(self, x, y): - scorer = getattr(self._best_estimator, self._scorer_name) - y_pred = scorer(x) - - input = self.drift_input(y, y_pred) - self.drift_detector.update(input) - - # We need to start the optimization process from scratch - if self.drift_detector.drift_detected: - self._n = 0 - self._converged = False - self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) - - # There is no proven best model right now - self._best_estimator = None - return - - self._best_estimator.learn_one(x, y) - - def _learn_not_converged(self, x, y): - for wrap in self._bandits: - scorer = getattr(wrap.estimator, self._scorer_name) - y_pred = scorer(x) - wrap.metric.update(y, y_pred) - wrap.estimator.learn_one(x, y) - - # Keep the simplex ordered - self._sort_bandits() - - if self._n == self.grace_period: - self._n = 0 - self._prune_bandits() - print("Nr bandits: ",len(self._bandits)) - # 1. Simplex in sphere - scaled_params_b = self._normalize_flattened_hyperspace( - self._bandits[0].estimator._get_params(), - ) - scaled_params_g = self._normalize_flattened_hyperspace( - self._bandits[1].estimator._get_params(), - ) - scaled_params_w = self._normalize_flattened_hyperspace( - self._bandits[2].estimator._get_params(), - ) - print("----------") - print( - "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric - ) - print( - "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric - ) - print( - "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric - ) - hyper_points = [ - list(scaled_params_b.values()), - list(scaled_params_g.values()), - list(scaled_params_w.values()), - ] - vectors = np.array(hyper_points) - self._old_centroid = dict( - zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) - ) - - - if self._models_converged: - self._converged = True - self._best_estimator = self._bandits[0].estimator - - def learn_one(self, x, y): - self._n += 1 - - if self.converged: - self._learn_converged(x, y) - else: - self._learn_not_converged(x, y) - - return self - - @property - def best(self): - if not self._converged: - # Lazy selection of the best model - self._sort_bandits() - return self._bandits[0].estimator - - return self._best_estimator - - @property - def converged(self): - return self._converged - - def predict_one(self, x, **kwargs): - try: - return self.best.predict_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_one' is not supported in {border.__class__.__name__}." - ) - - def predict_proba_one(self, x, **kwargs): - try: - return self.best.predict_proba_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_proba_one' is not supported in {border.__class__.__name__}." - ) - - def score_one(self, x, **kwargs): - try: - return self.best.score_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'score_one' is not supported in {border.__class__.__name__}." - ) - - def debug_one(self, x, **kwargs): - try: - return self.best.debug_one(x, **kwargs) - except NotImplementedError: - raise AttributeError( - f"'debug_one' is not supported in {self.best.__class__.__name__}." - ) diff --git a/river_extra/model_selection/bandit_g.py b/river_extra/model_selection/bandit_g.py deleted file mode 100644 index 9998ef7..0000000 --- a/river_extra/model_selection/bandit_g.py +++ /dev/null @@ -1,555 +0,0 @@ -import statistics -from ast import operator -import collections -import copy -import math -import numbers -import random -import typing -import pandas as pd -import numpy as np -from itertools import combinations -from tqdm import tqdm - -import numpy as np - -# TODO use lazy imports where needed -from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree - -ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") - - -class Bandit_ps(base.Estimator): - """Bandit Self Parameter Tuning - - Parameters - ---------- - estimator - metric - params_range - drift_input - grace_period - drift_detector - convergence_sphere - seed - - References - ---------- - [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning - for data streams. Information Fusion, 76, 75-86. - - """ - - _START_RANDOM = "random" - _START_WARM = "warm" - - def __init__( - self, - estimator: base.Estimator, - metric: metrics.base.Metric, - params_range: typing.Dict[str, typing.Tuple], - drift_input: typing.Callable[[float, float], float], - grace_period: int = 500, - drift_detector: base.DriftDetector = drift.ADWIN(), - convergence_sphere: float = 0.001, - nr_estimators: int = 50, - seed: int = None, - ): - super().__init__() - self.estimator = estimator - self.metric = metric - self.params_range = params_range - self.drift_input = drift_input - - self.grace_period = grace_period - self.drift_detector = drift_detector - self.convergence_sphere = convergence_sphere - - self.nr_estimators = nr_estimators - - self.seed = seed - - self._n = 0 - self._converged = False - self._rng = random.Random(self.seed) - grids = np.meshgrid(np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__]))) - self._sample=np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) - - self._df_hpi = pd.DataFrame(columns=['params', 'instances', 'score']) - - self._i=0 - self._p=0 - self._counter=0 - self._best_estimator = None - self._bandits = self._create_bandits(estimator,nr_estimators) - - # Convergence criterion - self._old_centroid = None - - self._pruned_configurations=[] - # Meta-programming - border = self.estimator - if isinstance(border, compose.Pipeline): - border = border[-1] - - if isinstance(border, (base.Classifier, base.Regressor)): - self._scorer_name = "predict_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "score_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "classify" - - def __generate(self, hp_data) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_type == int: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return int (val) - elif hp_type == float: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return val - - def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - - def __combine_bee(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_est_1>=hp_est_2: - func=lambda h1, h2: h2 + ((h1 + h2)*9 / 10) - else: - func = lambda h1, h2: h1 + ((h1 + h2)*9 / 10) - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - - def analyze_fanova(self, cal_df, max_iter=-1): - metric='score' - params = list(eval(cal_df.loc[0, 'params']).keys()) - result = pd.DataFrame(cal_df.loc[:, metric].copy()) - tmp_df = cal_df.loc[:, 'params'].copy() - for key in params: - result.insert(loc=0, column=key, value=tmp_df.apply(lambda x: eval(x)[key])) - col_name = params[:] - col_name.append(metric) - cal_df = result.reindex(columns=col_name).copy() - - axis = ['id'] - axis.extend(list(cal_df.columns)) - params = axis[1:-1] - metric = axis[-1] - f = pd.DataFrame(columns=axis) - f.loc[0, :] = np.nan - f.loc[0, metric] = cal_df[metric].mean() - f.loc[0, 'id'] = hash(str([])) - v_all = np.std(cal_df[metric].to_numpy()) ** 2 - v = pd.DataFrame(columns=['u', 'v_u', 'F_u(v_u/v_all)']) - for k in range(1, len(params) + 1): - for u in combinations(params, k): - ne_u = set(params) - set(u) - a_u = cal_df.groupby(list(u)).mean().reset_index() - if len(ne_u): - for nu in ne_u: - a_u.loc[:, nu] = np.nan - col_name = cal_df.columns.tolist() - a_u = a_u.reindex(columns=col_name) - sum_f_w = pd.DataFrame(columns=f.columns[:]) - tmp = [] - w_list = [] - for i in range(len(u)): - tmp.extend(list(combinations(u, i))) - for t in tmp: - w_list.append(list(t)) - - for w in w_list: - sum_f_w = pd.concat([sum_f_w, f[f['id'] == hash(str(w))]], ignore_index=True) - col_name = sum_f_w.columns.tolist() - a_u = a_u.reindex(columns=col_name) - for row_index, r in sum_f_w.iterrows(): - r2 = r[1:-1] - not_null_index = r2.notnull().values - if not not_null_index.any(): - a_u.loc[:, col_name[-1]] -= r[-1] - else: - left, right = a_u.iloc[:, 1:-1].align(r2[not_null_index], axis=1, copy=False) - equal_index = (left == right).values.sum(axis=1) - a_u.loc[equal_index == not_null_index.sum(), col_name[-1]] -= r[-1] - a_u['id'] = hash(str(list(u))) - f = pd.concat([f, a_u], ignore_index=True) - tmp_f_u = a_u.loc[:, metric].to_numpy() - v = pd.concat([v, pd.DataFrame( - [{'u': u, 'v_u': (tmp_f_u ** 2).mean(), 'F_u(v_u/v_all)': (tmp_f_u ** 2).mean() / v_all}])], - ignore_index=True) - - return v - - - def __flatten(self, prefix, scaled_hps, hp_data, est_data): - hp_range = hp_data[1] - interval = hp_range[1] - hp_range[0] - scaled_hps[prefix] = (est_data - hp_range[0]) / interval - - def _traverse_hps( - self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None - ) -> typing.Optional[typing.Union[dict, numbers.Number]]: - """Traverse the hyperparameters of the estimator/pipeline and perform an operation. - - Parameters - ---------- - operation - The operation that is intented to apply over the hyperparameters. Can be either: - "combine" (combine parameters from two pipelines), "scale" (scale a flattened - version of the hyperparameter hierarchy to use in the stopping criteria), or - "generate" (create a new hyperparameter set candidate). - hp_data - The hyperparameter data which was passed by the user. Defines the ranges to - explore for each hyperparameter. - est_1 - The hyperparameter structure of the first pipeline/estimator. Such structure is obtained - via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. - func - A function that is used to combine the values in `est_1` and `est_2`, in case - `operation="combine"`. - est_2 - A second pipeline/estimator which is going to be combined with `est_1`, in case - `operation="combine"`. - hp_prefix - A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter - hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly - to the current position in the hierarchy. Initially it is set to `None`. - scaled_hps - Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. - Set to `None` and defined automatically when `operation="scale"`. - """ - - # Sub-component needs to be instantiated - if isinstance(est_1, tuple): - sub_class, est_1 = est_1 - - if operation == "combine": - est_2 = est_2[1] - else: - est_2 = {} - - sub_config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name - else: - sub_hp_prefix = None - - sub_config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=est_1[sub_hp_name], - func=func, - est_2=est_2.get(sub_hp_name, None), - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - return sub_class(**sub_config) - - # We reached the numeric parameters - if isinstance(est_1, numbers.Number): - if operation == "generate": - return self.__generate(hp_data) - if operation == "scale": - self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) - return - # combine - return self.__combine(hp_data, est_1, est_2, func) - - # The sub-parameters need to be expanded - config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - sub_est_1 = est_1[sub_hp_name] - - if operation == "combine": - sub_est_2 = est_2[sub_hp_name] - else: - sub_est_2 = {} - - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name - else: - sub_hp_prefix = None - - config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=sub_est_1, - func=func, - est_2=sub_est_2, - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - - return config - - def _random_config(self): - return self._traverse_hps( - operation="generate", - hp_data=self.params_range, - est_1=self.estimator._get_params(), - ) - - def _create_bandits(self, model, nr_estimators) -> typing.List: - bandits = [None] * nr_estimators - - for i in range(nr_estimators): - bandits[i] = ModelWrapper( - model.clone(self._random_config(), include_attributes=True), - self.metric.clone(include_attributes=True), - ) - self._i+=1 - self._p=0 - self._i=0 - return bandits - - def _sort_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - if self.metric.bigger_is_better: - self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) - else: - self._bandits.sort(key=lambda mw: mw.metric.get()) - - def _prune_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - self._bandits[0].estimator._get_params() - half = int(1*len(self._bandits) / 10) - for i in range(len(self._bandits)): - #if i>half: - lst=list(self.params_range[self.estimator[1].__class__.__name__].keys()) - mydict={} - for x in lst: - mydict[x]=self._bandits[i].estimator._get_params()[self.estimator[1].__class__.__name__][x] - #mydict['instances'] - self._df_hpi.loc[len(self._df_hpi.index)] = [str(mydict), self._bandits[i].estimator[self.estimator[1].__class__.__name__].summary['total_observed_weight'], self._bandits[i].metric.get()] - #'{\'hp1\': ' + str(row[1]['hp1']) + ', \'hp2\': ' + str(row[1]['hp2']) + ', \'hp3\': ' + str(row[1]['hp3']) + '}' - #new_col = [] - #for row in df.iterrows(): - # new_col.append( - # ) - #df['params'] = new_col - self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ - str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) - #else: - #self._pruned_configurations.append( - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ - # 'grace_period']) + ',' + - # str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ - # 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) - - #if len(self._bandits)-half>2: - # self._bandits=self._bandits[:-half] - #else: - # self._bandits=self._bandits[:-(len(self._bandits)-3)] - - - def _gen_new_estimator(self, e1, e2, func): - """Generate new configuration given two estimators and a combination function.""" - - est_1_hps = e1.estimator._get_params() - est_2_hps = e2.estimator._get_params() - - new_config = self._traverse_hps( - operation="combine", - hp_data=self.params_range, - est_1=est_1_hps, - func=func, - est_2=est_2_hps - ) - - # Modify the current best contender with the new hyperparameter values - new = ModelWrapper( - copy.deepcopy(self._bandits[0].estimator), - self.metric.clone(include_attributes=True), - ) - new.estimator.mutate(new_config) - - return new - - - def _normalize_flattened_hyperspace(self, orig): - scaled = {} - self._traverse_hps( - operation="scale", - hp_data=self.params_range, - est_1=orig, - hp_prefix="", - scaled_hps=scaled - ) - return scaled - - @property - def _models_converged(self) -> bool: - if len(self._bandits)==3: - return True - else: - return False - - def _learn_converged(self, x, y): - scorer = getattr(self._best_estimator, self._scorer_name) - y_pred = scorer(x) - - input = self.drift_input(y, y_pred) - self.drift_detector.update(input) - - # We need to start the optimization process from scratch - if self.drift_detector.drift_detected: - self._n = 0 - self._converged = False - self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) - - # There is no proven best model right now - self._best_estimator = None - return - - self._best_estimator.learn_one(x, y) - - def _learn_not_converged(self, x, y): - for wrap in self._bandits: - scorer = getattr(wrap.estimator, self._scorer_name) - y_pred = scorer(x) - wrap.metric.update(y, y_pred) - wrap.estimator.learn_one(x, y) - - # Keep the simplex ordered - self._sort_bandits() - - if self._n == self.grace_period: - self._n = 0 - self._prune_bandits() - df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] - df3 = df3.reset_index() - importance=self.analyze_fanova(df3) - print(importance) - #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] - - #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] - print("Nr bandits: ",len(self._bandits)) - # 1. Simplex in sphere - scaled_params_b = self._normalize_flattened_hyperspace( - self._bandits[0].estimator._get_params(), - ) - scaled_params_g = self._normalize_flattened_hyperspace( - self._bandits[1].estimator._get_params(), - ) - scaled_params_w = self._normalize_flattened_hyperspace( - self._bandits[2].estimator._get_params(), - ) - #for i in range(1,len(self._bandits)): - # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) - - print("----------") - print( - "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric - ) - print( - "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric - ) - print( - "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric - ) - hyper_points = [ - list(scaled_params_b.values()), - list(scaled_params_g.values()), - list(scaled_params_w.values()), - ] - vectors = np.array(hyper_points) - self._old_centroid = dict( - zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) - ) - - - if self._models_converged: - self._converged = True - self._best_estimator = self._bandits[0].estimator - - def learn_one(self, x, y): - - - self._n += 1 - self._counter += 1 - - if self.converged: - self._learn_converged(x, y) - else: - self._learn_not_converged(x, y) - - return self - - @property - def best(self): - if not self._converged: - # Lazy selection of the best model - self._sort_bandits() - return self._bandits[0].estimator - - return self._best_estimator - - @property - def converged(self): - return self._converged - - def predict_one(self, x, **kwargs): - try: - return self.best.predict_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_one' is not supported in {border.__class__.__name__}." - ) - - def predict_proba_one(self, x, **kwargs): - try: - return self.best.predict_proba_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_proba_one' is not supported in {border.__class__.__name__}." - ) - - def score_one(self, x, **kwargs): - try: - return self.best.score_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'score_one' is not supported in {border.__class__.__name__}." - ) - - def debug_one(self, x, **kwargs): - try: - return self.best.debug_one(x, **kwargs) - except NotImplementedError: - raise AttributeError( - f"'debug_one' is not supported in {self.best.__class__.__name__}." - ) diff --git a/river_extra/model_selection/benchmark_classification.py b/river_extra/model_selection/benchmark_classification.py deleted file mode 100644 index 9657ebc..0000000 --- a/river_extra/model_selection/benchmark_classification.py +++ /dev/null @@ -1,145 +0,0 @@ -import matplotlib.pyplot as plt -from river import datasets, drift, metrics, preprocessing, tree, utils -from river.datasets import synth - -from river_extra.model_selection.bandit_g import Bandit_ps -from river_extra.model_selection.sspt import SSPT -from river_extra.model_selection.bandit_d import Bandit - -# Dataset -dataset = datasets.synth.ConceptDriftStream( - stream=synth.SEA(seed=42, variant=0), - drift_stream=synth.SEA(seed=40, variant=1), - seed=1, - position=10000, - width=2, -).take(20000) - -from river import datasets -#dataset = synth.Hyperplane(seed=42, n_features=2).take(20000) -dataset = datasets.CreditCard().take(20000) - -# Baseline - model and metric -baseline_metric = metrics.Accuracy() -baseline_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) -baseline_metric_plt = [] -baseline_rolling_metric_plt = [] -baseline = preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier() - -# SSPT - model and metric -sspt_metric = metrics.Accuracy() -sspt_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) -sspt_metric_plt = [] -sspt_rolling_metric_plt = [] -sspt = SSPT( - estimator=preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier(), - metric=sspt_rolling_metric, - grace_period=1000, - params_range={ - #"AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, - "HoeffdingTreeClassifier": { - "tau": (float, (0.01, 0.09)), - "delta": (float, (0.00000001, 0.000001)), - "grace_period": (int, (100, 500)), - }, - }, - drift_detector=drift.ADWIN(), - drift_input=lambda yt, yp: 0 if yt == yp else 1, - convergence_sphere=0.000001, - seed=42, -) - -bandit_metric = metrics.Accuracy() -bandit_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) -bandit_metric_plt = [] -bandit_rolling_metric_plt = [] -bandit = Bandit_ps( - estimator=preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier(), - metric=bandit_rolling_metric, - grace_period=1000, - params_range={ - #"AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, - "HoeffdingTreeClassifier": { - "tau": (float, (0.01, 0.1)), - "delta": (float, (0.00001, 0.001)), - "grace_period": (int, (100, 1000)), - "fake_hp": (float, (0.01, 0.1)), - }, - }, - drift_detector=drift.ADWIN(), - drift_input=lambda yt, yp: 0 if yt == yp else 1, - convergence_sphere=0.000001, - nr_estimators=64, - seed=42, -) - -sspt_first_print = True -bandit_first_print = True - -for i, (x, y) in enumerate(dataset): - if i%1000==0: - print(i) - baseline_y_pred = baseline.predict_one(x) - baseline_metric.update(y, baseline_y_pred) - baseline_rolling_metric.update(y, baseline_y_pred) - baseline_metric_plt.append(baseline_metric.get()) - baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) - baseline.learn_one(x, y) - #print('Baseline-------------') - #print(baseline2.debug_one(x)) - - - - - sspt_y_pred = sspt.predict_one(x) - sspt_metric.update(y, sspt_y_pred) - sspt_rolling_metric.update(y, sspt_y_pred) - sspt_metric_plt.append(sspt_metric.get()) - sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) - sspt.learn_one(x, y) - #print('Bandit-------------') - #print(bandit.debug_one(x)) - bandit_y_pred = bandit.predict_one(x) - bandit_metric.update(y, bandit_y_pred) - bandit_rolling_metric.update(y, bandit_y_pred) - bandit_metric_plt.append(bandit_metric.get()) - bandit_rolling_metric_plt.append(bandit_rolling_metric.get()) - bandit.learn_one(x, y) - if sspt.converged and sspt_first_print: - print("SSPT Converged at:", i) - sspt_first_print = False - if bandit.converged and bandit_first_print: - print("Bandit Converged at:", i) - bandit_first_print = False - if sspt.drift_detector.drift_detected: - print("SSPT Drift Detected at:",i) - if bandit.drift_detector.drift_detected: - print("Bandit Drift Detected at:",i) - - -print("Total instances:", i + 1) -print(repr(baseline)) -print("SSPT Best params:") -print(repr(sspt.best)) -print("Bandit Best params:") -print(repr(bandit.best)) -print("SSPT: ", sspt_metric) -print("Bandits: ", bandit_metric) -print("Baseline: ", baseline_metric) - - - -plt.plot(baseline_metric_plt[:20000], linestyle="dotted") -plt.plot(sspt_metric_plt[:20000]) -plt.show() - -plt.plot(baseline_rolling_metric_plt[0:40000],'k') -plt.plot(bandit_rolling_metric_plt[0:40000], 'r') -plt.plot(sspt_rolling_metric_plt[0:40000], 'b') -plt.show() - -with open('/Users/brunoveloso/Downloads/ensaio11.csv', 'w') as f: - - for i in bandit._pruned_configurations: - f.write(i+'\n') - diff --git a/river_extra/model_selection/classification_test.py b/river_extra/model_selection/classification_test.py deleted file mode 100644 index 030687c..0000000 --- a/river_extra/model_selection/classification_test.py +++ /dev/null @@ -1,193 +0,0 @@ -import math - -import matplotlib.pyplot as plt -import river -from river import datasets, drift, metrics, preprocessing, tree, utils -from river.datasets import synth - -from river_extra.model_selection.bandit_g import Bandit_ps -from river_extra.model_selection.sspt import SSPT -from river_extra.model_selection.bandit_d import Bandit - -def myfunc(val): - # Dataset - dataset = river.datasets.synth.ConceptDriftStream( - stream=synth.SEA(seed=42, variant=0), - drift_stream=synth.SEA(seed=40, variant=1), - seed=1, - position=10000, - width=2, - ).take(20000) - - from river import datasets - #dataset = synth.Hyperplane(seed=42, n_features=2).take(20000) - dataset = datasets.CreditCard().take(20000) - - # Baseline - model and metric - baseline_metric = metrics.Accuracy() - baseline_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) - baseline_metric_plt = [] - baseline_rolling_metric_plt = [] - baseline = preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier() - - # SSPT - model and metric - sspt_metric = metrics.Accuracy() - sspt_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) - sspt_metric_plt = [] - sspt_rolling_metric_plt = [] - sspt = SSPT( - estimator=preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier(), - metric=sspt_rolling_metric, - grace_period=1000, - params_range={ - #"AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, - "HoeffdingTreeClassifier": { - "tau": (float, (0.01, 0.09)), - "delta": (float, (0.00000001, 0.000001)), - "grace_period": (int, (100, 500)), - }, - }, - drift_detector=drift.ADWIN(), - drift_input=lambda yt, yp: 0 if yt == yp else 1, - convergence_sphere=0.000001, - seed=42, - ) - - bandit_metric = metrics.Accuracy() - bandit_rolling_metric = utils.Rolling(metrics.Accuracy(), window_size=1000) - bandit_metric_plt = [] - bandit_rolling_metric_plt = [] - bandit = Bandit_ps( - estimator=preprocessing.StandardScaler()|tree.HoeffdingTreeClassifier(), - metric=bandit_rolling_metric, - grace_period=1000, - params_range={ - #"AdaptiveStandardScaler": {"alpha": (float, (0.25, 0.35))}, - "HoeffdingTreeClassifier": { - "tau": (float, (0.01, 0.1)), - "delta": (float, (0.00001, 0.001)), - "grace_period": (int, (100, 1000)), - }, - }, - drift_detector=drift.ADWIN(), - drift_input=lambda yt, yp: 0 if yt == yp else 1, - convergence_sphere=0.000001, - nr_estimators=64, - seed=42, - ) - - sspt_first_print = True - bandit_first_print = True - # given parameters by user - we_epsilon = 0.1 - we_delta = 0.1 - we_miu = 0 - we_sig2_sum = 0 - we_dev2_sum = 0 - we_dev2 = 0 - we_sig2 = 0 - dyn_w=[] - dyn_n=[] - sig2=[] - dev2=[] - vari = 0 - for i, (x, y) in enumerate(dataset): - if i%1000==0: - print(i) - baseline_y_pred = baseline.predict_one(x) - baseline_metric.update(y, baseline_y_pred) - baseline_rolling_metric.update(y, baseline_y_pred) - baseline_metric_plt.append(baseline_metric.get()) - baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) - baseline.learn_one(x, y) - #print('Baseline-------------') - #print(baseline2.debug_one(x)) - - - - we_lambda = (math.e - 2) - we_upsilon = 4 * we_lambda * math.log(2 / we_delta) / math.pow(we_epsilon, 2) - old_miu=we_miu - we_miu = we_miu+(baseline_rolling_metric.get()-we_miu)/(i+1) - vari = vari + (baseline_rolling_metric.get() - old_miu) * (baseline_rolling_metric.get() - we_miu) - stdn = math.sqrt(vari / (i + 1)) - - we_sig2_sum+=math.pow(baseline_rolling_metric.get(),2) - we_dev2_sum += baseline_rolling_metric.get() - - we_dev2 = (1/(i+1))*we_sig2_sum - math.pow((1/(i+1)*we_dev2_sum),2) - dev2.append((stdn)) - we_n = math.pow(1.96, 2) * stdn*(1-stdn)/(math.pow(0.01,2)) - dyn_n.append(we_n) - - we_sig2 = we_sig2_sum-(i+1)*math.pow(we_miu,2) - sig2.append(we_sig2) - we_ro = max(we_sig2, we_epsilon * we_miu) - if we_miu>0: - we_n = we_upsilon * we_ro / math.pow(we_miu, 2) - dyn_w.append(we_n) - print(we_n) - - # sspt_y_pred = sspt.predict_one(x) - # sspt_metric.update(y, sspt_y_pred) - # sspt_rolling_metric.update(y, sspt_y_pred) - # sspt_metric_plt.append(sspt_metric.get()) - # sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) - # sspt.learn_one(x, y) - # #print('Bandit-------------') - # #print(bandit.debug_one(x)) - # bandit_y_pred = bandit.predict_one(x) - # bandit_metric.update(y, bandit_y_pred) - # bandit_rolling_metric.update(y, bandit_y_pred) - # bandit_metric_plt.append(bandit_metric.get()) - # bandit_rolling_metric_plt.append(bandit_rolling_metric.get()) - # bandit.learn_one(x, y) - # if sspt.converged and sspt_first_print: - # print("SSPT Converged at:", i) - # sspt_first_print = False - # if bandit.converged and bandit_first_print: - # print("Bandit Converged at:", i) - # bandit_first_print = False - # if sspt.drift_detector.drift_detected: - # print("SSPT Drift Detected at:",i) - # if bandit.drift_detector.drift_detected: - # print("Bandit Drift Detected at:",i) - - - print("Total instances:", i + 1) - print(repr(baseline)) - print("SSPT Best params:") - print(repr(sspt.best)) - print("Bandit Best params:") - print(repr(bandit.best)) - print("SSPT: ", sspt_metric) - print("Bandits: ", bandit_metric) - print("Baseline: ", baseline_metric) - - plt.plot(dyn_w, linestyle="dotted") - #plt.plot(dyn_n) - plt.show() - plt.plot(baseline_rolling_metric_plt[500:]) - plt.show() - - # plt.plot(baseline_metric_plt[:20000], linestyle="dotted") - # plt.plot(sspt_metric_plt[:20000]) - # plt.show() - # - # plt.plot(baseline_rolling_metric_plt[0:40000],'k') - # plt.plot(bandit_rolling_metric_plt[0:40000], 'r') - # plt.plot(sspt_rolling_metric_plt[0:40000], 'b') - # plt.show() - - with open('/Users/brunoveloso/Downloads/ensaio11.csv', 'w') as f: - - for i in bandit._pruned_configurations: - f.write(i+'\n') - - - -from bigO import BigO - -lib=BigO() -complexity=lib.test(myfunc,"random") -print(complexity) \ No newline at end of file diff --git a/river_extra/model_selection/functional_anova.py b/river_extra/model_selection/functional_anova.py deleted file mode 100644 index c5b7425..0000000 --- a/river_extra/model_selection/functional_anova.py +++ /dev/null @@ -1,536 +0,0 @@ -import statistics -from ast import operator -import collections -import copy -import math -import numbers -import random -import typing -import pandas as pd -import numpy as np -from itertools import combinations -from tqdm import tqdm - -import numpy as np - -# TODO use lazy imports where needed -from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree - -ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") - - -class Functional_Anova(base.Estimator): - """Bandit Self Parameter Tuning - - Parameters - ---------- - estimator - metric - params_range - drift_input - grace_period - drift_detector - convergence_sphere - seed - - References - ---------- - [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning - for data streams. Information Fusion, 76, 75-86. - - """ - - _START_RANDOM = "random" - _START_WARM = "warm" - - def __init__( - self, - estimator: base.Estimator, - metric: metrics.base.Metric, - params_range: typing.Dict[str, typing.Tuple], - drift_input: typing.Callable[[float, float], float], - grace_period: int = 500, - drift_detector: base.DriftDetector = drift.ADWIN(), - convergence_sphere: float = 0.001, - nr_estimators: int = 50, - seed: int = None, - ): - super().__init__() - self.estimator = estimator - self.metric = metric - self.params_range = params_range - self.drift_input = drift_input - - self.grace_period = grace_period - self.drift_detector = drift_detector - self.convergence_sphere = convergence_sphere - - self.nr_estimators = nr_estimators - - self.seed = seed - - self._n = 0 - self._converged = False - self._rng = random.Random(self.seed) - grids = np.meshgrid(np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__]))) - self._sample=np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) - - self._df_hpi = pd.DataFrame(columns=['params', 'instances', 'score']) - - self._i=0 - self._p=0 - self._counter=0 - self._best_estimator = None - self._bandits = self._create_bandits(estimator,nr_estimators) - - # Convergence criterion - self._old_centroid = None - - self._pruned_configurations=[] - # Meta-programming - border = self.estimator - if isinstance(border, compose.Pipeline): - border = border[-1] - - if isinstance(border, (base.Classifier, base.Regressor)): - self._scorer_name = "predict_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "score_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "classify" - - def __generate(self, hp_data) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_type == int: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return int (val) - elif hp_type == float: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return val - - def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - - def analyze_fanova(self, cal_df, max_iter=-1): - metric='score' - params = list(eval(cal_df.loc[0, 'params']).keys()) - result = pd.DataFrame(cal_df.loc[:, metric].copy()) - tmp_df = cal_df.loc[:, 'params'].copy() - for key in params: - result.insert(loc=0, column=key, value=tmp_df.apply(lambda x: eval(x)[key])) - col_name = params[:] - col_name.append(metric) - cal_df = result.reindex(columns=col_name).copy() - - axis = ['id'] - axis.extend(list(cal_df.columns)) - params = axis[1:-1] - metric = axis[-1] - f = pd.DataFrame(columns=axis) - f.loc[0, :] = np.nan - f.loc[0, metric] = cal_df[metric].mean() - f.loc[0, 'id'] = hash(str([])) - v_all = np.std(cal_df[metric].to_numpy()) ** 2 - v = pd.DataFrame(columns=['u', 'v_u', 'F_u(v_u/v_all)']) - for k in range(1, len(params) + 1): - for u in combinations(params, k): - ne_u = set(params) - set(u) - a_u = cal_df.groupby(list(u)).mean().reset_index() - if len(ne_u): - for nu in ne_u: - a_u.loc[:, nu] = np.nan - col_name = cal_df.columns.tolist() - a_u = a_u.reindex(columns=col_name) - sum_f_w = pd.DataFrame(columns=f.columns[:]) - tmp = [] - w_list = [] - for i in range(len(u)): - tmp.extend(list(combinations(u, i))) - for t in tmp: - w_list.append(list(t)) - - for w in w_list: - sum_f_w = pd.concat([sum_f_w, f[f['id'] == hash(str(w))]], ignore_index=True) - col_name = sum_f_w.columns.tolist() - a_u = a_u.reindex(columns=col_name) - for row_index, r in sum_f_w.iterrows(): - r2 = r[1:-1] - not_null_index = r2.notnull().values - if not not_null_index.any(): - a_u.loc[:, col_name[-1]] -= r[-1] - else: - left, right = a_u.iloc[:, 1:-1].align(r2[not_null_index], axis=1, copy=False) - equal_index = (left == right).values.sum(axis=1) - a_u.loc[equal_index == not_null_index.sum(), col_name[-1]] -= r[-1] - a_u['id'] = hash(str(list(u))) - f = pd.concat([f, a_u], ignore_index=True) - tmp_f_u = a_u.loc[:, metric].to_numpy() - v = pd.concat([v, pd.DataFrame( - [{'u': u, 'v_u': (tmp_f_u ** 2).mean(), 'F_u(v_u/v_all)': (tmp_f_u ** 2).mean() / v_all}])], - ignore_index=True) - - return v - - - def __flatten(self, prefix, scaled_hps, hp_data, est_data): - hp_range = hp_data[1] - interval = hp_range[1] - hp_range[0] - scaled_hps[prefix] = (est_data - hp_range[0]) / interval - - def _traverse_hps( - self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None - ) -> typing.Optional[typing.Union[dict, numbers.Number]]: - """Traverse the hyperparameters of the estimator/pipeline and perform an operation. - - Parameters - ---------- - operation - The operation that is intented to apply over the hyperparameters. Can be either: - "combine" (combine parameters from two pipelines), "scale" (scale a flattened - version of the hyperparameter hierarchy to use in the stopping criteria), or - "generate" (create a new hyperparameter set candidate). - hp_data - The hyperparameter data which was passed by the user. Defines the ranges to - explore for each hyperparameter. - est_1 - The hyperparameter structure of the first pipeline/estimator. Such structure is obtained - via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. - func - A function that is used to combine the values in `est_1` and `est_2`, in case - `operation="combine"`. - est_2 - A second pipeline/estimator which is going to be combined with `est_1`, in case - `operation="combine"`. - hp_prefix - A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter - hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly - to the current position in the hierarchy. Initially it is set to `None`. - scaled_hps - Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. - Set to `None` and defined automatically when `operation="scale"`. - """ - - # Sub-component needs to be instantiated - if isinstance(est_1, tuple): - sub_class, est_1 = est_1 - - if operation == "combine": - est_2 = est_2[1] - else: - est_2 = {} - - sub_config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name - else: - sub_hp_prefix = None - - sub_config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=est_1[sub_hp_name], - func=func, - est_2=est_2.get(sub_hp_name, None), - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - return sub_class(**sub_config) - - # We reached the numeric parameters - if isinstance(est_1, numbers.Number): - if operation == "generate": - return self.__generate(hp_data) - if operation == "scale": - self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) - return - # combine - return self.__combine(hp_data, est_1, est_2, func) - - # The sub-parameters need to be expanded - config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - sub_est_1 = est_1[sub_hp_name] - - if operation == "combine": - sub_est_2 = est_2[sub_hp_name] - else: - sub_est_2 = {} - - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name - else: - sub_hp_prefix = None - - config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=sub_est_1, - func=func, - est_2=sub_est_2, - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - - return config - - def _random_config(self): - return self._traverse_hps( - operation="generate", - hp_data=self.params_range, - est_1=self.estimator._get_params(), - ) - - def _create_bandits(self, model, nr_estimators) -> typing.List: - bandits = [None] * nr_estimators - - for i in range(nr_estimators): - bandits[i] = ModelWrapper( - model.clone(self._random_config(), include_attributes=True), - self.metric.clone(include_attributes=True), - ) - self._i+=1 - self._p=0 - self._i=0 - return bandits - - def _sort_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - if self.metric.bigger_is_better: - self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) - else: - self._bandits.sort(key=lambda mw: mw.metric.get()) - - def _prune_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - self._bandits[0].estimator._get_params() - half = int(1*len(self._bandits) / 10) - for i in range(len(self._bandits)): - #if i>half: - lst=list(self.params_range[self.estimator[1].__class__.__name__].keys()) - mydict={} - for x in lst: - mydict[x]=self._bandits[i].estimator._get_params()[self.estimator[1].__class__.__name__][x] - #mydict['instances'] - self._df_hpi.loc[len(self._df_hpi.index)] = [str(mydict), self._bandits[i].estimator[self.estimator[1].__class__.__name__].summary['total_observed_weight'], self._bandits[i].metric.get()] - #'{\'hp1\': ' + str(row[1]['hp1']) + ', \'hp2\': ' + str(row[1]['hp2']) + ', \'hp3\': ' + str(row[1]['hp3']) + '}' - #new_col = [] - #for row in df.iterrows(): - # new_col.append( - # ) - #df['params'] = new_col - self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ - str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) - #else: - #self._pruned_configurations.append( - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ - # 'grace_period']) + ',' + - # str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ - # 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) - - #if len(self._bandits)-half>2: - # self._bandits=self._bandits[:-half] - #else: - # self._bandits=self._bandits[:-(len(self._bandits)-3)] - - - def _gen_new_estimator(self, e1, e2, func): - """Generate new configuration given two estimators and a combination function.""" - - est_1_hps = e1.estimator._get_params() - est_2_hps = e2.estimator._get_params() - - new_config = self._traverse_hps( - operation="combine", - hp_data=self.params_range, - est_1=est_1_hps, - func=func, - est_2=est_2_hps - ) - - # Modify the current best contender with the new hyperparameter values - new = ModelWrapper( - copy.deepcopy(self._bandits[0].estimator), - self.metric.clone(include_attributes=True), - ) - new.estimator.mutate(new_config) - - return new - - - def _normalize_flattened_hyperspace(self, orig): - scaled = {} - self._traverse_hps( - operation="scale", - hp_data=self.params_range, - est_1=orig, - hp_prefix="", - scaled_hps=scaled - ) - return scaled - - @property - def _models_converged(self) -> bool: - if len(self._bandits)==3: - return True - else: - return False - - def _learn_converged(self, x, y): - scorer = getattr(self._best_estimator, self._scorer_name) - y_pred = scorer(x) - - input = self.drift_input(y, y_pred) - self.drift_detector.update(input) - - # We need to start the optimization process from scratch - if self.drift_detector.drift_detected: - self._n = 0 - self._converged = False - self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) - - # There is no proven best model right now - self._best_estimator = None - return - - self._best_estimator.learn_one(x, y) - - def _learn_not_converged(self, x, y): - for wrap in self._bandits: - scorer = getattr(wrap.estimator, self._scorer_name) - y_pred = scorer(x) - wrap.metric.update(y, y_pred) - wrap.estimator.learn_one(x, y) - - # Keep the simplex ordered - self._sort_bandits() - - if self._n == self.grace_period: - self._n = 0 - self._prune_bandits() - df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] - df3 = df3.reset_index() - importance=self.analyze_fanova(df3) - print(importance) - #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] - - #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] - print("Nr bandits: ",len(self._bandits)) - # 1. Simplex in sphere - scaled_params_b = self._normalize_flattened_hyperspace( - self._bandits[0].estimator._get_params(), - ) - scaled_params_g = self._normalize_flattened_hyperspace( - self._bandits[1].estimator._get_params(), - ) - scaled_params_w = self._normalize_flattened_hyperspace( - self._bandits[2].estimator._get_params(), - ) - #for i in range(1,len(self._bandits)): - # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) - - print("----------") - print( - "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric - ) - print( - "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric - ) - print( - "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric - ) - hyper_points = [ - list(scaled_params_b.values()), - list(scaled_params_g.values()), - list(scaled_params_w.values()), - ] - vectors = np.array(hyper_points) - self._old_centroid = dict( - zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) - ) - - - if self._models_converged: - self._converged = True - self._best_estimator = self._bandits[0].estimator - - def learn_one(self, x, y): - self._n += 1 - self._counter += 1 - - if self.converged: - self._learn_converged(x, y) - else: - self._learn_not_converged(x, y) - - return self - - @property - def best(self): - if not self._converged: - # Lazy selection of the best model - self._sort_bandits() - return self._bandits[0].estimator - - return self._best_estimator - - @property - def converged(self): - return self._converged - - def predict_one(self, x, **kwargs): - try: - return self.best.predict_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_one' is not supported in {border.__class__.__name__}." - ) - - def predict_proba_one(self, x, **kwargs): - try: - return self.best.predict_proba_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_proba_one' is not supported in {border.__class__.__name__}." - ) - - def score_one(self, x, **kwargs): - try: - return self.best.score_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'score_one' is not supported in {border.__class__.__name__}." - ) - - def debug_one(self, x, **kwargs): - try: - return self.best.debug_one(x, **kwargs) - except NotImplementedError: - raise AttributeError( - f"'debug_one' is not supported in {self.best.__class__.__name__}." - ) diff --git a/river_extra/model_selection/grid_search.py b/river_extra/model_selection/grid_search.py deleted file mode 100644 index 7e8e84e..0000000 --- a/river_extra/model_selection/grid_search.py +++ /dev/null @@ -1,472 +0,0 @@ -import statistics -from ast import operator -import collections -import copy -import math -import numbers -import random -import typing -import pandas as pd -import numpy as np -from itertools import combinations -from tqdm import tqdm - -import numpy as np - -# TODO use lazy imports where needed -from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree - -ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") - - -class Grid_Search(base.Estimator): - """Bandit Self Parameter Tuning - - Parameters - ---------- - estimator - metric - params_range - drift_input - grace_period - drift_detector - convergence_sphere - seed - - References - ---------- - [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning - for data streams. Information Fusion, 76, 75-86. - - """ - - _START_RANDOM = "random" - _START_WARM = "warm" - - def __init__( - self, - estimator: base.Estimator, - metric: metrics.base.Metric, - params_range: typing.Dict[str, typing.Tuple], - drift_input: typing.Callable[[float, float], float], - grace_period: int = 500, - drift_detector: base.DriftDetector = drift.ADWIN(), - convergence_sphere: float = 0.001, - nr_estimators: int = 50, - seed: int = None, - ): - super().__init__() - self.estimator = estimator - self.metric = metric - self.params_range = params_range - self.drift_input = drift_input - - self.grace_period = grace_period - self.drift_detector = drift_detector - self.convergence_sphere = convergence_sphere - - self.nr_estimators = nr_estimators - - self.seed = seed - - self._n = 0 - self._converged = False - self._rng = random.Random(self.seed) - grids = np.meshgrid(np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__]))) - self._sample=np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) - - self._i=0 - self._p=0 - self._counter=0 - self._best_estimator = None - self._bandits = self._create_bandits(estimator,nr_estimators) - - # Convergence criterion - self._old_centroid = None - - # Meta-programming - border = self.estimator - if isinstance(border, compose.Pipeline): - border = border[-1] - - if isinstance(border, (base.Classifier, base.Regressor)): - self._scorer_name = "predict_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "score_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "classify" - - def __generate(self, hp_data) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_type == int: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return int (val) - elif hp_type == float: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return val - - def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - - def __flatten(self, prefix, scaled_hps, hp_data, est_data): - hp_range = hp_data[1] - interval = hp_range[1] - hp_range[0] - scaled_hps[prefix] = (est_data - hp_range[0]) / interval - - def _traverse_hps( - self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None - ) -> typing.Optional[typing.Union[dict, numbers.Number]]: - """Traverse the hyperparameters of the estimator/pipeline and perform an operation. - - Parameters - ---------- - operation - The operation that is intented to apply over the hyperparameters. Can be either: - "combine" (combine parameters from two pipelines), "scale" (scale a flattened - version of the hyperparameter hierarchy to use in the stopping criteria), or - "generate" (create a new hyperparameter set candidate). - hp_data - The hyperparameter data which was passed by the user. Defines the ranges to - explore for each hyperparameter. - est_1 - The hyperparameter structure of the first pipeline/estimator. Such structure is obtained - via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. - func - A function that is used to combine the values in `est_1` and `est_2`, in case - `operation="combine"`. - est_2 - A second pipeline/estimator which is going to be combined with `est_1`, in case - `operation="combine"`. - hp_prefix - A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter - hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly - to the current position in the hierarchy. Initially it is set to `None`. - scaled_hps - Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. - Set to `None` and defined automatically when `operation="scale"`. - """ - - # Sub-component needs to be instantiated - if isinstance(est_1, tuple): - sub_class, est_1 = est_1 - - if operation == "combine": - est_2 = est_2[1] - else: - est_2 = {} - - sub_config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name - else: - sub_hp_prefix = None - - sub_config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=est_1[sub_hp_name], - func=func, - est_2=est_2.get(sub_hp_name, None), - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - return sub_class(**sub_config) - - # We reached the numeric parameters - if isinstance(est_1, numbers.Number): - if operation == "generate": - return self.__generate(hp_data) - if operation == "scale": - self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) - return - # combine - return self.__combine(hp_data, est_1, est_2, func) - - # The sub-parameters need to be expanded - config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - sub_est_1 = est_1[sub_hp_name] - - if operation == "combine": - sub_est_2 = est_2[sub_hp_name] - else: - sub_est_2 = {} - - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name - else: - sub_hp_prefix = None - - config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=sub_est_1, - func=func, - est_2=sub_est_2, - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - - return config - - def _random_config(self): - return self._traverse_hps( - operation="generate", - hp_data=self.params_range, - est_1=self.estimator._get_params(), - ) - - def _create_bandits(self, model, nr_estimators) -> typing.List: - bandits = [None] * nr_estimators - - for i in range(nr_estimators): - bandits[i] = ModelWrapper( - model.clone(self._random_config(), include_attributes=True), - self.metric.clone(include_attributes=True), - ) - self._i+=1 - self._p=0 - self._i=0 - return bandits - - def _sort_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - if self.metric.bigger_is_better: - self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) - else: - self._bandits.sort(key=lambda mw: mw.metric.get()) - - def _prune_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - self._bandits[0].estimator._get_params() - half = int(1*len(self._bandits) / 10) - for i in range(len(self._bandits)): - #if i>half: - lst=list(self.params_range[self.estimator[1].__class__.__name__].keys()) - mydict={} - for x in lst: - mydict[x]=self._bandits[i].estimator._get_params()[self.estimator[1].__class__.__name__][x] - #mydict['instances'] - self._df_hpi.loc[len(self._df_hpi.index)] = [str(mydict), self._bandits[i].estimator[self.estimator[1].__class__.__name__].summary['total_observed_weight'], self._bandits[i].metric.get()] - #'{\'hp1\': ' + str(row[1]['hp1']) + ', \'hp2\': ' + str(row[1]['hp2']) + ', \'hp3\': ' + str(row[1]['hp3']) + '}' - #new_col = [] - #for row in df.iterrows(): - # new_col.append( - # ) - #df['params'] = new_col - self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ - str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) - #else: - #self._pruned_configurations.append( - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ - # 'grace_period']) + ',' + - # str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ - # 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) - - #if len(self._bandits)-half>2: - # self._bandits=self._bandits[:-half] - #else: - # self._bandits=self._bandits[:-(len(self._bandits)-3)] - - - def _gen_new_estimator(self, e1, e2, func): - """Generate new configuration given two estimators and a combination function.""" - - est_1_hps = e1.estimator._get_params() - est_2_hps = e2.estimator._get_params() - - new_config = self._traverse_hps( - operation="combine", - hp_data=self.params_range, - est_1=est_1_hps, - func=func, - est_2=est_2_hps - ) - - # Modify the current best contender with the new hyperparameter values - new = ModelWrapper( - copy.deepcopy(self._bandits[0].estimator), - self.metric.clone(include_attributes=True), - ) - new.estimator.mutate(new_config) - - return new - - - def _normalize_flattened_hyperspace(self, orig): - scaled = {} - self._traverse_hps( - operation="scale", - hp_data=self.params_range, - est_1=orig, - hp_prefix="", - scaled_hps=scaled - ) - return scaled - - @property - def _models_converged(self) -> bool: - if len(self._bandits)==3: - return True - else: - return False - - def _learn_converged(self, x, y): - scorer = getattr(self._best_estimator, self._scorer_name) - y_pred = scorer(x) - - input = self.drift_input(y, y_pred) - self.drift_detector.update(input) - - # We need to start the optimization process from scratch - if self.drift_detector.drift_detected: - self._n = 0 - self._converged = False - self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) - - # There is no proven best model right now - self._best_estimator = None - return - - self._best_estimator.learn_one(x, y) - - def _learn_not_converged(self, x, y): - for wrap in self._bandits: - scorer = getattr(wrap.estimator, self._scorer_name) - y_pred = scorer(x) - wrap.metric.update(y, y_pred) - wrap.estimator.learn_one(x, y) - - # Keep the simplex ordered - self._sort_bandits() - - if self._n == self.grace_period: - self._n = 0 - self._prune_bandits() - df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] - df3 = df3.reset_index() - importance=self.analyze_fanova(df3) - print(importance) - #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] - - #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] - print("Nr bandits: ",len(self._bandits)) - # 1. Simplex in sphere - scaled_params_b = self._normalize_flattened_hyperspace( - self._bandits[0].estimator._get_params(), - ) - scaled_params_g = self._normalize_flattened_hyperspace( - self._bandits[1].estimator._get_params(), - ) - scaled_params_w = self._normalize_flattened_hyperspace( - self._bandits[2].estimator._get_params(), - ) - #for i in range(1,len(self._bandits)): - # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) - - print("----------") - print( - "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric - ) - print( - "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric - ) - print( - "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric - ) - hyper_points = [ - list(scaled_params_b.values()), - list(scaled_params_g.values()), - list(scaled_params_w.values()), - ] - vectors = np.array(hyper_points) - self._old_centroid = dict( - zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) - ) - - - if self._models_converged: - self._converged = True - self._best_estimator = self._bandits[0].estimator - - def learn_one(self, x, y): - self._n += 1 - self._counter += 1 - - if self.converged: - self._learn_converged(x, y) - else: - self._learn_not_converged(x, y) - - return self - - @property - def best(self): - if not self._converged: - # Lazy selection of the best model - self._sort_bandits() - return self._bandits[0].estimator - - return self._best_estimator - - @property - def converged(self): - return self._converged - - def predict_one(self, x, **kwargs): - try: - return self.best.predict_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_one' is not supported in {border.__class__.__name__}." - ) - - def predict_proba_one(self, x, **kwargs): - try: - return self.best.predict_proba_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_proba_one' is not supported in {border.__class__.__name__}." - ) - - def score_one(self, x, **kwargs): - try: - return self.best.score_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'score_one' is not supported in {border.__class__.__name__}." - ) - - def debug_one(self, x, **kwargs): - try: - return self.best.debug_one(x, **kwargs) - except NotImplementedError: - raise AttributeError( - f"'debug_one' is not supported in {self.best.__class__.__name__}." - ) diff --git a/river_extra/model_selection/hyper_band.py b/river_extra/model_selection/hyper_band.py deleted file mode 100644 index 4286a30..0000000 --- a/river_extra/model_selection/hyper_band.py +++ /dev/null @@ -1,437 +0,0 @@ -import collections -import copy -import numbers -import random -import typing - -import numpy as np -# TODO use lazy imports where needed -from river import anomaly, base, compose, drift, metrics -from scipy.stats import qmc - -ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") - - -class Hyper_Band(base.Estimator): - """Bandit Self Parameter Tuning - - Parameters - ---------- - estimator - metric - params_range - drift_input - grace_period - drift_detector - convergence_sphere - seed - - References - ---------- - - """ - - _START_RANDOM = "random" - _START_WARM = "warm" - - def __init__( - self, - estimator: base.Estimator, - metric: metrics.base.Metric, - params_range: typing.Dict[str, typing.Tuple], - drift_input: typing.Callable[[float, float], float], - grace_period: int = 500, - drift_detector: base.DriftDetector = drift.ADWIN(), - convergence_sphere: float = 0.001, - nr_estimators: int = 50, - seed: int = None, - ): - super().__init__() - self.estimator = estimator - self.metric = metric - self.params_range = params_range - self.drift_input = drift_input - - self.grace_period = grace_period - self.drift_detector = drift_detector - self.convergence_sphere = convergence_sphere - - self.nr_estimators = nr_estimators - - self.seed = seed - - self._n = 0 - self._converged = False - self._rng = random.Random(self.seed) - - sampler = qmc.LatinHypercube(d=len(params_range[self.estimator[1].__class__.__name__])) - self._sample = sampler.random(n=nr_estimators) - self._i=0 - self._p=0 - self._counter=0 - self._best_estimator = None - self._bandits = self._create_bandits(estimator,nr_estimators) - - # Convergence criterion - self._old_centroid = None - - self._pruned_configurations=[] - # Meta-programming - border = self.estimator - if isinstance(border, compose.Pipeline): - border = border[-1] - - if isinstance(border, (base.Classifier, base.Regressor)): - self._scorer_name = "predict_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "score_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "classify" - - def __generate(self, hp_data) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_type == int: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return int (val) - elif hp_type == float: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return val - - def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - - def __flatten(self, prefix, scaled_hps, hp_data, est_data): - hp_range = hp_data[1] - interval = hp_range[1] - hp_range[0] - scaled_hps[prefix] = (est_data - hp_range[0]) / interval - - def _traverse_hps( - self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None - ) -> typing.Optional[typing.Union[dict, numbers.Number]]: - """Traverse the hyperparameters of the estimator/pipeline and perform an operation. - - Parameters - ---------- - operation - The operation that is intented to apply over the hyperparameters. Can be either: - "combine" (combine parameters from two pipelines), "scale" (scale a flattened - version of the hyperparameter hierarchy to use in the stopping criteria), or - "generate" (create a new hyperparameter set candidate). - hp_data - The hyperparameter data which was passed by the user. Defines the ranges to - explore for each hyperparameter. - est_1 - The hyperparameter structure of the first pipeline/estimator. Such structure is obtained - via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. - func - A function that is used to combine the values in `est_1` and `est_2`, in case - `operation="combine"`. - est_2 - A second pipeline/estimator which is going to be combined with `est_1`, in case - `operation="combine"`. - hp_prefix - A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter - hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly - to the current position in the hierarchy. Initially it is set to `None`. - scaled_hps - Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. - Set to `None` and defined automatically when `operation="scale"`. - """ - - # Sub-component needs to be instantiated - if isinstance(est_1, tuple): - sub_class, est_1 = est_1 - - if operation == "combine": - est_2 = est_2[1] - else: - est_2 = {} - - sub_config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name - else: - sub_hp_prefix = None - - sub_config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=est_1[sub_hp_name], - func=func, - est_2=est_2.get(sub_hp_name, None), - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - return sub_class(**sub_config) - - # We reached the numeric parameters - if isinstance(est_1, numbers.Number): - if operation == "generate": - return self.__generate(hp_data) - if operation == "scale": - self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) - return - # combine - return self.__combine(hp_data, est_1, est_2, func) - - # The sub-parameters need to be expanded - config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - sub_est_1 = est_1[sub_hp_name] - - if operation == "combine": - sub_est_2 = est_2[sub_hp_name] - else: - sub_est_2 = {} - - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name - else: - sub_hp_prefix = None - - config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=sub_est_1, - func=func, - est_2=sub_est_2, - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - - return config - - def _random_config(self): - return self._traverse_hps( - operation="generate", - hp_data=self.params_range, - est_1=self.estimator._get_params(), - ) - - def _create_bandits(self, model, nr_estimators) -> typing.List: - bandits = [None] * nr_estimators - - for i in range(nr_estimators): - bandits[i] = ModelWrapper( - model.clone(self._random_config(), include_attributes=True), - self.metric.clone(include_attributes=True), - ) - self._i+=1 - self._p=0 - self._i=0 - return bandits - - def _sort_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - if self.metric.bigger_is_better: - self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) - else: - self._bandits.sort(key=lambda mw: mw.metric.get()) - - def _prune_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - self._bandits[0].estimator._get_params() - half = int(len(self._bandits) /2) - if len(self._bandits)-half>1: - self._bandits=self._bandits[:-half] - else: - self._bandits=self._bandits[:-(len(self._bandits)-1)] - - - def _gen_new_estimator(self, e1, e2, func): - """Generate new configuration given two estimators and a combination function.""" - - est_1_hps = e1.estimator._get_params() - est_2_hps = e2.estimator._get_params() - - new_config = self._traverse_hps( - operation="combine", - hp_data=self.params_range, - est_1=est_1_hps, - func=func, - est_2=est_2_hps - ) - - # Modify the current best contender with the new hyperparameter values - new = ModelWrapper( - copy.deepcopy(self._bandits[0].estimator), - self.metric.clone(include_attributes=True), - ) - new.estimator.mutate(new_config) - - return new - - - def _normalize_flattened_hyperspace(self, orig): - scaled = {} - self._traverse_hps( - operation="scale", - hp_data=self.params_range, - est_1=orig, - hp_prefix="", - scaled_hps=scaled - ) - return scaled - - @property - def _models_converged(self) -> bool: - if len(self._bandits)==3: - return True - else: - return False - - def _learn_converged(self, x, y): - scorer = getattr(self._best_estimator, self._scorer_name) - y_pred = scorer(x) - - input = self.drift_input(y, y_pred) - self.drift_detector.update(input) - - # We need to start the optimization process from scratch - if self.drift_detector.drift_detected: - self._n = 0 - self._converged = False - self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) - - # There is no proven best model right now - self._best_estimator = None - return - - self._best_estimator.learn_one(x, y) - - def _learn_not_converged(self, x, y): - for wrap in self._bandits: - scorer = getattr(wrap.estimator, self._scorer_name) - y_pred = scorer(x) - wrap.metric.update(y, y_pred) - wrap.estimator.learn_one(x, y) - - # Keep the simplex ordered - self._sort_bandits() - - if self._n == self.grace_period: - self._n = 0 - self._prune_bandits() - df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] - df3 = df3.reset_index() - importance=self.analyze_fanova(df3) - print(importance) - #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] - - #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] - print("Nr bandits: ",len(self._bandits)) - # 1. Simplex in sphere - scaled_params_b = self._normalize_flattened_hyperspace( - self._bandits[0].estimator._get_params(), - ) - scaled_params_g = self._normalize_flattened_hyperspace( - self._bandits[1].estimator._get_params(), - ) - scaled_params_w = self._normalize_flattened_hyperspace( - self._bandits[2].estimator._get_params(), - ) - #for i in range(1,len(self._bandits)): - # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) - - print("----------") - print( - "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric - ) - print( - "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric - ) - print( - "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric - ) - hyper_points = [ - list(scaled_params_b.values()), - list(scaled_params_g.values()), - list(scaled_params_w.values()), - ] - vectors = np.array(hyper_points) - self._old_centroid = dict( - zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) - ) - - - if self._models_converged: - self._converged = True - self._best_estimator = self._bandits[0].estimator - - def learn_one(self, x, y): - self._n += 1 - self._counter += 1 - - if self.converged: - self._learn_converged(x, y) - else: - self._learn_not_converged(x, y) - - return self - - @property - def best(self): - if not self._converged: - # Lazy selection of the best model - self._sort_bandits() - return self._bandits[0].estimator - - return self._best_estimator - - @property - def converged(self): - return self._converged - - def predict_one(self, x, **kwargs): - try: - return self.best.predict_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_one' is not supported in {border.__class__.__name__}." - ) - - def predict_proba_one(self, x, **kwargs): - try: - return self.best.predict_proba_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_proba_one' is not supported in {border.__class__.__name__}." - ) - - def score_one(self, x, **kwargs): - try: - return self.best.score_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'score_one' is not supported in {border.__class__.__name__}." - ) - - def debug_one(self, x, **kwargs): - try: - return self.best.debug_one(x, **kwargs) - except NotImplementedError: - raise AttributeError( - f"'debug_one' is not supported in {self.best.__class__.__name__}." - ) diff --git a/river_extra/model_selection/importance.csv b/river_extra/model_selection/importance.csv deleted file mode 100644 index 6e50a3d..0000000 --- a/river_extra/model_selection/importance.csv +++ /dev/null @@ -1,4 +0,0 @@ -,u,v_u,F_u(v_u/v_all) -0,"('alpha',)",0.05688453514739228,0.8920573509330904 -1,"('l1_ratio',)",0.00248888888888889,0.03903049613108786 -2,"('alpha', 'l1_ratio')",0.004394376417233563,0.06891215293582162 diff --git a/river_extra/model_selection/iris[GridSearchCV]Model1.csv b/river_extra/model_selection/iris[GridSearchCV]Model1.csv deleted file mode 100644 index 0b7e0b0..0000000 --- a/river_extra/model_selection/iris[GridSearchCV]Model1.csv +++ /dev/null @@ -1,26 +0,0 @@ -,mean_fit_time,std_fit_time,mean_score_time,std_score_time,param_alpha,param_l1_ratio,params,split0_test_score,split1_test_score,split2_test_score,mean_test_score,std_test_score,rank_test_score -0,0.0038988590240478516,0.00019350259080068593,0.048513333002726235,0.007620761818638512,0.0009765625,0.0,"{'alpha': 0.0009765625, 'l1_ratio': 0.0}",0.8285714285714286,0.9714285714285714,0.9714285714285714,0.9238095238095237,0.06734350297014735,4 -1,0.0034006436665852866,0.0005841867367244808,0.042453765869140625,0.011295176044315678,0.0009765625,0.25,"{'alpha': 0.0009765625, 'l1_ratio': 0.25}",0.8857142857142857,0.9714285714285714,0.9428571428571428,0.9333333333333332,0.03563483225498993,3 -2,0.002705812454223633,0.0005023765548274845,0.04854400952657064,0.009058915009452781,0.0009765625,0.5,"{'alpha': 0.0009765625, 'l1_ratio': 0.5}",0.8857142857142857,1.0,0.9428571428571428,0.9428571428571427,0.04665694748158436,1 -3,0.0033037662506103516,0.0005313131511568678,0.04070862134297689,0.0030310135946994684,0.0009765625,0.75,"{'alpha': 0.0009765625, 'l1_ratio': 0.75}",0.8857142857142857,0.9142857142857143,0.9142857142857143,0.9047619047619047,0.01346870059402948,5 -4,0.0018006960550944011,0.00011594471970534648,0.0002894401550292969,1.3626756826625067e-05,0.0009765625,1.0,"{'alpha': 0.0009765625, 'l1_ratio': 1.0}",0.8857142857142857,1.0,0.9428571428571428,0.9428571428571427,0.04665694748158436,1 -5,0.002614736557006836,0.0006447643344046145,0.00036334991455078125,6.784366740978792e-05,0.03125,0.0,"{'alpha': 0.03125, 'l1_ratio': 0.0}",0.7428571428571429,0.9428571428571428,0.9142857142857143,0.8666666666666667,0.08832017614757812,6 -6,0.002805948257446289,0.00011343430505195735,0.0003763834635416667,1.5144532979525363e-05,0.03125,0.25,"{'alpha': 0.03125, 'l1_ratio': 0.25}",0.7714285714285715,0.8857142857142857,0.9142857142857143,0.8571428571428571,0.06172133998483673,8 -7,0.0027430057525634766,0.00018951542775052176,0.0003916422526041667,4.8637909564400674e-05,0.03125,0.5,"{'alpha': 0.03125, 'l1_ratio': 0.5}",0.7428571428571429,0.9142857142857143,0.9142857142857143,0.8571428571428571,0.08081220356417683,8 -8,0.0027420520782470703,0.0002221225341207598,0.0003730456034342448,1.4321441516853438e-05,0.03125,0.75,"{'alpha': 0.03125, 'l1_ratio': 0.75}",0.8,0.8285714285714286,0.9142857142857143,0.8476190476190476,0.048562090605645536,10 -9,0.0027184486389160156,0.0001826884858045175,0.00036780039469401043,2.4055740185533338e-05,0.03125,1.0,"{'alpha': 0.03125, 'l1_ratio': 1.0}",0.8285714285714286,0.8857142857142857,0.8857142857142857,0.8666666666666667,0.02693740118805891,6 -10,0.002456823984781901,0.0003652350772010518,0.00036676724751790363,6.140912660445521e-05,1.0,0.0,"{'alpha': 1.0, 'l1_ratio': 0.0}",0.7142857142857143,0.6857142857142857,0.7142857142857143,0.7047619047619048,0.01346870059402948,11 -11,0.002367417017618815,0.0004354789732352187,0.0003585020701090495,7.064605302385832e-05,1.0,0.25,"{'alpha': 1.0, 'l1_ratio': 0.25}",0.6857142857142857,0.6857142857142857,0.7142857142857143,0.6952380952380953,0.01346870059402948,12 -12,0.0028084119160970054,5.444453551458701e-05,0.0003689130147298177,8.18145975216623e-06,1.0,0.5,"{'alpha': 1.0, 'l1_ratio': 0.5}",0.6857142857142857,0.6857142857142857,0.45714285714285713,0.6095238095238095,0.10774960475223583,13 -13,0.0017545223236083984,9.122868150470193e-05,0.00028316179911295575,2.7951182098326422e-05,1.0,0.75,"{'alpha': 1.0, 'l1_ratio': 0.75}",0.3142857142857143,0.37142857142857144,0.34285714285714286,0.3428571428571428,0.02332847374079218,20 -14,0.0028853416442871094,0.00012350005685968156,0.00037741661071777344,7.251871230888679e-05,1.0,1.0,"{'alpha': 1.0, 'l1_ratio': 1.0}",0.3142857142857143,0.3142857142857143,0.37142857142857144,0.3333333333333333,0.02693740118805896,21 -15,0.0020644664764404297,0.0005100051870199706,0.0003059705098470052,3.298822581682996e-05,32.0,0.0,"{'alpha': 32.0, 'l1_ratio': 0.0}",0.37142857142857144,0.3142857142857143,0.7142857142857143,0.4666666666666666,0.17664035229515626,15 -16,0.002279361089070638,0.0003502445116967939,0.00038584073384602863,7.185155265996111e-06,32.0,0.25,"{'alpha': 32.0, 'l1_ratio': 0.25}",0.37142857142857144,0.37142857142857144,0.37142857142857144,0.37142857142857144,0.0,16 -17,0.0024990240732828775,0.0004403886677270806,0.0003437201182047526,4.276422691293242e-05,32.0,0.5,"{'alpha': 32.0, 'l1_ratio': 0.5}",0.3142857142857143,0.37142857142857144,0.37142857142857144,0.3523809523809524,0.026937401188058964,18 -18,0.002029975255330404,0.00037354139523400955,0.0003112951914469401,4.8085086434913315e-05,32.0,0.75,"{'alpha': 32.0, 'l1_ratio': 0.75}",0.3142857142857143,0.37142857142857144,0.37142857142857144,0.3523809523809524,0.026937401188058964,18 -19,0.002227306365966797,0.00039614039127220334,0.0003238519032796224,5.381304453676507e-05,32.0,1.0,"{'alpha': 32.0, 'l1_ratio': 1.0}",0.37142857142857144,0.37142857142857144,0.37142857142857144,0.37142857142857144,0.0,16 -20,0.0023507277170817056,0.0005203862341675505,0.0003177324930826823,3.9434718072432945e-05,1024.0,0.0,"{'alpha': 1024.0, 'l1_ratio': 0.0}",0.7714285714285715,0.37142857142857144,0.34285714285714286,0.49523809523809526,0.19564417699213466,14 -21,0.002264738082885742,0.00041749469149180644,0.0003310839335123698,3.608995478017171e-05,1024.0,0.25,"{'alpha': 1024.0, 'l1_ratio': 0.25}",0.3142857142857143,0.3142857142857143,0.37142857142857144,0.3333333333333333,0.02693740118805896,21 -22,0.0018083254496256511,0.0001286763033791477,0.00031304359436035156,4.640942831202071e-05,1024.0,0.5,"{'alpha': 1024.0, 'l1_ratio': 0.5}",0.3142857142857143,0.37142857142857144,0.2857142857142857,0.3238095238095238,0.03563483225498993,24 -23,0.0018104712168375652,0.00016673333992452716,0.00029699007670084637,5.461479925405132e-05,1024.0,0.75,"{'alpha': 1024.0, 'l1_ratio': 0.75}",0.3142857142857143,0.3142857142857143,0.37142857142857144,0.3333333333333333,0.02693740118805896,21 -24,0.0021660327911376953,0.0001316696688384735,0.00031280517578125,5.5347355293830595e-05,1024.0,1.0,"{'alpha': 1024.0, 'l1_ratio': 1.0}",0.3142857142857143,0.3142857142857143,0.2857142857142857,0.30476190476190473,0.01346870059402948,25 diff --git a/river_extra/model_selection/random_search.py b/river_extra/model_selection/random_search.py deleted file mode 100644 index 113c985..0000000 --- a/river_extra/model_selection/random_search.py +++ /dev/null @@ -1,479 +0,0 @@ -import statistics -from ast import operator -import collections -import copy -import math -import numbers -import random -import typing -import pandas as pd -import numpy as np -from itertools import combinations - -from scipy.stats import qmc -from tqdm import tqdm - -import numpy as np - -# TODO use lazy imports where needed -from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree - -ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") - - -class Random_Search(base.Estimator): - """Bandit Self Parameter Tuning - - Parameters - ---------- - estimator - metric - params_range - drift_input - grace_period - drift_detector - convergence_sphere - seed - - References - ---------- - [1]: Veloso, B., Gama, J., Malheiro, B., & Vinagre, J. (2021). Hyperparameter self-tuning - for data streams. Information Fusion, 76, 75-86. - - """ - - _START_RANDOM = "random" - _START_WARM = "warm" - - def __init__( - self, - estimator: base.Estimator, - metric: metrics.base.Metric, - params_range: typing.Dict[str, typing.Tuple], - drift_input: typing.Callable[[float, float], float], - grace_period: int = 500, - drift_detector: base.DriftDetector = drift.ADWIN(), - convergence_sphere: float = 0.001, - nr_estimators: int = 50, - seed: int = None, - ): - super().__init__() - self.estimator = estimator - self.metric = metric - self.params_range = params_range - self.drift_input = drift_input - - self.grace_period = grace_period - self.drift_detector = drift_detector - self.convergence_sphere = convergence_sphere - - self.nr_estimators = nr_estimators - - self.seed = seed - - self._n = 0 - self._converged = False - self._rng = random.Random(self.seed) - grids = np.meshgrid(np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__])), np.linspace(0,1,len(params_range[self.estimator[1].__class__.__name__]))) - self._sample=np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) - - self._df_hpi = pd.DataFrame(columns=['params', 'instances', 'score']) - - sampler = qmc.LatinHypercube(d=len(params_range[self.estimator[1].__class__.__name__])) - self._sample = sampler.random(n=nr_estimators) - self._i=0 - self._p=0 - self._counter=0 - self._best_estimator = None - self._bandits = self._create_bandits(estimator,nr_estimators) - - # Convergence criterion - self._old_centroid = None - - self._pruned_configurations=[] - # Meta-programming - border = self.estimator - if isinstance(border, compose.Pipeline): - border = border[-1] - - if isinstance(border, (base.Classifier, base.Regressor)): - self._scorer_name = "predict_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "score_one" - elif isinstance(border, anomaly.base.AnomalyDetector): - self._scorer_name = "classify" - - def __generate(self, hp_data) -> numbers.Number: - hp_type, hp_range = hp_data - if hp_type == int: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return int (val) - elif hp_type == float: - val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] - self._p += 1 - return val - - def __combine(self, hp_data, hp_est_1, hp_est_2, func) -> numbers.Number: - hp_type, hp_range = hp_data - new_val = func(hp_est_1, hp_est_2) - - # Range sanity checks - if new_val < hp_range[0]: - new_val = hp_range[0] - if new_val > hp_range[1]: - new_val = hp_range[1] - - new_val = round(new_val, 0) if hp_type == int else new_val - return new_val - - def __flatten(self, prefix, scaled_hps, hp_data, est_data): - hp_range = hp_data[1] - interval = hp_range[1] - hp_range[0] - scaled_hps[prefix] = (est_data - hp_range[0]) / interval - - def _traverse_hps( - self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None - ) -> typing.Optional[typing.Union[dict, numbers.Number]]: - """Traverse the hyperparameters of the estimator/pipeline and perform an operation. - - Parameters - ---------- - operation - The operation that is intented to apply over the hyperparameters. Can be either: - "combine" (combine parameters from two pipelines), "scale" (scale a flattened - version of the hyperparameter hierarchy to use in the stopping criteria), or - "generate" (create a new hyperparameter set candidate). - hp_data - The hyperparameter data which was passed by the user. Defines the ranges to - explore for each hyperparameter. - est_1 - The hyperparameter structure of the first pipeline/estimator. Such structure is obtained - via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. - func - A function that is used to combine the values in `est_1` and `est_2`, in case - `operation="combine"`. - est_2 - A second pipeline/estimator which is going to be combined with `est_1`, in case - `operation="combine"`. - hp_prefix - A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter - hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly - to the current position in the hierarchy. Initially it is set to `None`. - scaled_hps - Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. - Set to `None` and defined automatically when `operation="scale"`. - """ - - # Sub-component needs to be instantiated - if isinstance(est_1, tuple): - sub_class, est_1 = est_1 - - if operation == "combine": - est_2 = est_2[1] - else: - est_2 = {} - - sub_config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name - else: - sub_hp_prefix = None - - sub_config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=est_1[sub_hp_name], - func=func, - est_2=est_2.get(sub_hp_name, None), - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - return sub_class(**sub_config) - - # We reached the numeric parameters - if isinstance(est_1, numbers.Number): - if operation == "generate": - return self.__generate(hp_data) - if operation == "scale": - self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) - return - # combine - return self.__combine(hp_data, est_1, est_2, func) - - # The sub-parameters need to be expanded - config = {} - for sub_hp_name, sub_hp_data in hp_data.items(): - sub_est_1 = est_1[sub_hp_name] - - if operation == "combine": - sub_est_2 = est_2[sub_hp_name] - else: - sub_est_2 = {} - - if operation == "scale": - sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name - else: - sub_hp_prefix = None - - config[sub_hp_name] = self._traverse_hps( - operation=operation, - hp_data=sub_hp_data, - est_1=sub_est_1, - func=func, - est_2=sub_est_2, - hp_prefix=sub_hp_prefix, - scaled_hps=scaled_hps, - ) - - return config - - def _random_config(self): - return self._traverse_hps( - operation="generate", - hp_data=self.params_range, - est_1=self.estimator._get_params(), - ) - - def _create_bandits(self, model, nr_estimators) -> typing.List: - bandits = [None] * nr_estimators - - for i in range(nr_estimators): - bandits[i] = ModelWrapper( - model.clone(self._random_config(), include_attributes=True), - self.metric.clone(include_attributes=True), - ) - self._i+=1 - self._p=0 - self._i=0 - return bandits - - def _sort_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - if self.metric.bigger_is_better: - self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) - else: - self._bandits.sort(key=lambda mw: mw.metric.get()) - - def _prune_bandits(self): - """Ensure the simplex models are ordered by predictive performance.""" - self._bandits[0].estimator._get_params() - half = int(1*len(self._bandits) / 10) - for i in range(len(self._bandits)): - #if i>half: - lst=list(self.params_range[self.estimator[1].__class__.__name__].keys()) - mydict={} - for x in lst: - mydict[x]=self._bandits[i].estimator._get_params()[self.estimator[1].__class__.__name__][x] - #mydict['instances'] - self._df_hpi.loc[len(self._df_hpi.index)] = [str(mydict), self._bandits[i].estimator[self.estimator[1].__class__.__name__].summary['total_observed_weight'], self._bandits[i].metric.get()] - #'{\'hp1\': ' + str(row[1]['hp1']) + ', \'hp2\': ' + str(row[1]['hp2']) + ', \'hp3\': ' + str(row[1]['hp3']) + '}' - #new_col = [] - #for row in df.iterrows(): - # new_col.append( - # ) - #df['params'] = new_col - self._pruned_configurations.append(str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau'])+','+ - str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['grace_period'])+','+ - str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary['total_observed_weight'])+','+str(self._bandits[i].metric.get())) - #else: - #self._pruned_configurations.append( - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['delta']) + ',' + - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier']['tau']) + ',' + - # str(self._bandits[i].estimator._get_params()['HoeffdingTreeClassifier'][ - # 'grace_period']) + ',' + - # str(self._bandits[i].estimator['HoeffdingTreeClassifier'].summary[ - # 'total_observed_weight']) + ',' + str(self._bandits[i].metric.get())) - - #if len(self._bandits)-half>2: - # self._bandits=self._bandits[:-half] - #else: - # self._bandits=self._bandits[:-(len(self._bandits)-3)] - - - def _gen_new_estimator(self, e1, e2, func): - """Generate new configuration given two estimators and a combination function.""" - - est_1_hps = e1.estimator._get_params() - est_2_hps = e2.estimator._get_params() - - new_config = self._traverse_hps( - operation="combine", - hp_data=self.params_range, - est_1=est_1_hps, - func=func, - est_2=est_2_hps - ) - - # Modify the current best contender with the new hyperparameter values - new = ModelWrapper( - copy.deepcopy(self._bandits[0].estimator), - self.metric.clone(include_attributes=True), - ) - new.estimator.mutate(new_config) - - return new - - - def _normalize_flattened_hyperspace(self, orig): - scaled = {} - self._traverse_hps( - operation="scale", - hp_data=self.params_range, - est_1=orig, - hp_prefix="", - scaled_hps=scaled - ) - return scaled - - @property - def _models_converged(self) -> bool: - if len(self._bandits)==3: - return True - else: - return False - - def _learn_converged(self, x, y): - scorer = getattr(self._best_estimator, self._scorer_name) - y_pred = scorer(x) - - input = self.drift_input(y, y_pred) - self.drift_detector.update(input) - - # We need to start the optimization process from scratch - if self.drift_detector.drift_detected: - self._n = 0 - self._converged = False - self._bandits = self._create_bandits(self._best_estimator, self.nr_estimators) - - # There is no proven best model right now - self._best_estimator = None - return - - self._best_estimator.learn_one(x, y) - - def _learn_not_converged(self, x, y): - for wrap in self._bandits: - scorer = getattr(wrap.estimator, self._scorer_name) - y_pred = scorer(x) - wrap.metric.update(y, y_pred) - wrap.estimator.learn_one(x, y) - - # Keep the simplex ordered - self._sort_bandits() - - if self._n == self.grace_period: - self._n = 0 - self._prune_bandits() - df3 = self._df_hpi[self._df_hpi['instances'] ==self._counter] - df3 = df3.reset_index() - importance=self.analyze_fanova(df3) - print(importance) - #hpi=importance[importance['F_u(v_u/v_all)']==max(importance['F_u(v_u/v_all)'])]['u'][0] - - #importance[importance['F_u(v_u/v_all)'] == max(importance['F_u(v_u/v_all)'])] - print("Nr bandits: ",len(self._bandits)) - # 1. Simplex in sphere - scaled_params_b = self._normalize_flattened_hyperspace( - self._bandits[0].estimator._get_params(), - ) - scaled_params_g = self._normalize_flattened_hyperspace( - self._bandits[1].estimator._get_params(), - ) - scaled_params_w = self._normalize_flattened_hyperspace( - self._bandits[2].estimator._get_params(), - ) - #for i in range(1,len(self._bandits)): - # self._bandits[i]=self._gen_new_estimator(self._bandits[0],self._bandits[i],lambda h1, h2: ((h1 + h2)*3) / 4) - - print("----------") - print( - "B:", list(scaled_params_b.values()), "Score:", self._bandits[0].metric - ) - print( - "G:", list(scaled_params_g.values()), "Score:", self._bandits[1].metric - ) - print( - "W:", list(scaled_params_w.values()), "Score:", self._bandits[2].metric - ) - hyper_points = [ - list(scaled_params_b.values()), - list(scaled_params_g.values()), - list(scaled_params_w.values()), - ] - vectors = np.array(hyper_points) - self._old_centroid = dict( - zip(scaled_params_b.keys(), np.mean(vectors, axis=0)) - ) - - - if self._models_converged: - self._converged = True - self._best_estimator = self._bandits[0].estimator - - def learn_one(self, x, y): - self._n += 1 - self._counter += 1 - - if self.converged: - self._learn_converged(x, y) - else: - self._learn_not_converged(x, y) - - return self - - @property - def best(self): - if not self._converged: - # Lazy selection of the best model - self._sort_bandits() - return self._bandits[0].estimator - - return self._best_estimator - - @property - def converged(self): - return self._converged - - def predict_one(self, x, **kwargs): - try: - return self.best.predict_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_one' is not supported in {border.__class__.__name__}." - ) - - def predict_proba_one(self, x, **kwargs): - try: - return self.best.predict_proba_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'predict_proba_one' is not supported in {border.__class__.__name__}." - ) - - def score_one(self, x, **kwargs): - try: - return self.best.score_one(x, **kwargs) - except NotImplementedError: - border = self.best - if isinstance(border, compose.Pipeline): - border = border[-1] - raise AttributeError( - f"'score_one' is not supported in {border.__class__.__name__}." - ) - - def debug_one(self, x, **kwargs): - try: - return self.best.debug_one(x, **kwargs) - except NotImplementedError: - raise AttributeError( - f"'debug_one' is not supported in {self.best.__class__.__name__}." - ) diff --git a/river_extra/model_selection/regression_test.py b/river_extra/model_selection/regression_test.py deleted file mode 100644 index 3b22445..0000000 --- a/river_extra/model_selection/regression_test.py +++ /dev/null @@ -1,79 +0,0 @@ -import matplotlib.pyplot as plt -from river import datasets, drift, linear_model, metrics, preprocessing, utils, rules - -from river_extra import model_selection - -# Dataset -dataset = datasets.synth.FriedmanDrift( - drift_type="gra", position=(7000, 9000), seed=42 -).take(20000) - - -# Baseline - model and metric -baseline_metric = metrics.RMSE() -baseline_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) -baseline_metric_plt = [] -baseline_rolling_metric_plt = [] -baseline = preprocessing.MinMaxScaler() | rules.AMRules() - -# SSPT - model and metric -sspt_metric = metrics.RMSE() -sspt_rolling_metric = utils.Rolling(metrics.RMSE(), window_size=100) -sspt_metric_plt = [] -sspt_rolling_metric_plt = [] -sspt = model_selection.SSPT( - estimator=preprocessing.MinMaxScaler() | rules.AMRules(), - metric=metrics.RMSE(), - grace_period=100, - params_range={ - #"MinMaxScaler": {"alpha": (float, (0.25, 0.35))}, - "AMRules": { - "tau": (float, (0.01, 0.1)), - "delta": (float, (0.00001, 0.001)), - "n_min": (int, (50, 500)) - }, - }, - drift_input=lambda yt, yp: abs(yt - yp), - drift_detector=drift.PageHinkley(), - convergence_sphere=0.000001, - seed=42, -) - -first_print = True - -metric = metrics.RMSE() - - -for i, (x, y) in enumerate(dataset): - baseline_y_pred = baseline.predict_one(x) - baseline_metric.update(y, baseline_y_pred) - baseline_rolling_metric.update(y, baseline_y_pred) - baseline_metric_plt.append(baseline_metric.get()) - baseline_rolling_metric_plt.append(baseline_rolling_metric.get()) - baseline.learn_one(x, y) - sspt_y_pred = sspt.predict_one(x) - sspt_metric.update(y, sspt_y_pred) - sspt_rolling_metric.update(y, sspt_y_pred) - sspt_metric_plt.append(sspt_metric.get()) - sspt_rolling_metric_plt.append(sspt_rolling_metric.get()) - sspt.learn_one(x, y) - - if sspt.converged and first_print: - print("Converged at:", i) - first_print = False - -print("Total instances:", i + 1) -print(repr(baseline)) -print("Best params:") -print(repr(sspt.best)) -print("SSPT: ", sspt_metric) -print("Baseline: ", baseline_metric) - - -#plt.plot(baseline_metric_plt[:10000], linestyle="dotted") -#plt.plot(sspt_metric_plt[:10000]) -#plt.show() - -plt.plot(baseline_rolling_metric_plt[:20000], linestyle="dotted") -plt.plot(sspt_rolling_metric_plt[:20000]) -plt.show() diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 1654b1f..2058af7 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -12,7 +12,6 @@ ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") - class SSPT(base.Estimator): """Single-pass Self Parameter Tuning @@ -85,6 +84,7 @@ def __init__( elif isinstance(border, anomaly.base.AnomalyDetector): self._scorer_name = "classify" + def __generate(self, hp_data) -> numbers.Number: hp_type, hp_range = hp_data if hp_type == int: diff --git a/river_extra/model_selection/teste.ipynb b/river_extra/model_selection/teste.ipynb deleted file mode 100644 index 6a33c37..0000000 --- a/river_extra/model_selection/teste.ipynb +++ /dev/null @@ -1,57 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true, - "ExecuteTime": { - "end_time": "2023-05-12T15:38:06.947205Z", - "start_time": "2023-05-12T15:38:06.927809Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "teste\n" - ] - } - ], - "source": [ - "print('teste')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/river_extra/model_selection/teste.py b/river_extra/model_selection/teste.py deleted file mode 100644 index b30fd82..0000000 --- a/river_extra/model_selection/teste.py +++ /dev/null @@ -1,25 +0,0 @@ -import hyanova - -import pandas as pd -import numpy as np -from itertools import combinations -from tqdm import tqdm - - - - -df = pd.read_csv('/Users/brunoveloso/Downloads/ensaio11.csv', names=['hp1', 'hp2', 'hp3', 'instances', 'score']) -print(df) - - - -for i in df.instances.unique(): - print('------->' + str(i)) - df3=df[df['instances']==i] - df3=df3.reset_index() - df2, params = hyanova.read_df(df3, 'score') - #print(df2['score'].unique()) - importance = analyze_incr(df2,max_iter=-1) - print(importance) - #break - From 53e2b525e9776258a3942e158bb312077bfcb690 Mon Sep 17 00:00:00 2001 From: BrunoMVeloso Date: Wed, 17 Apr 2024 16:07:26 +0100 Subject: [PATCH 50/50] V2 Added Grid Search Added Random Search Added MESSPT - Moya, A. R., Veloso, B., Gama, J., & Ventura, S. (2023). Improving hyper-parameter self-tuning for data streams by adapting an evolutionary approach. Data Mining and Knowledge Discovery, 1-27. --- river_extra/model_selection/grid_search.py | 310 +++++++ river_extra/model_selection/messpt.py | 863 +++++++++++++++++++ river_extra/model_selection/random_search.py | 296 +++++++ river_extra/model_selection/sspt.py | 4 +- 4 files changed, 1471 insertions(+), 2 deletions(-) create mode 100644 river_extra/model_selection/grid_search.py create mode 100644 river_extra/model_selection/messpt.py create mode 100644 river_extra/model_selection/random_search.py diff --git a/river_extra/model_selection/grid_search.py b/river_extra/model_selection/grid_search.py new file mode 100644 index 0000000..fa0ee1a --- /dev/null +++ b/river_extra/model_selection/grid_search.py @@ -0,0 +1,310 @@ +import statistics +from ast import operator +import collections +import copy +import math +import numbers +import random +import typing +import pandas as pd +import numpy as np +from itertools import combinations + +from scipy.stats import qmc +from tqdm import tqdm + +import numpy as np + +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + +class Grid_Search(base.Estimator): + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + nr_estimators: int = 50, + seed: int = None, + ): + super().__init__() + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + + self.nr_estimators = nr_estimators + + self.seed = seed + + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + arr_lin=[] + for param in params_range[self.estimator[1].__class__.__name__]: + arr_lin.append(np.linspace(params_range[self.estimator[1].__class__.__name__][param][1][0],params_range[self.estimator[1].__class__.__name__][param][1][1],round(self.nr_estimators**(1./3.)))) + grids = np.meshgrid(arr_lin[0],arr_lin[1],arr_lin[2]) + self._sample = np.moveaxis(np.array(grids), 0, grids[0].ndim).reshape(-1, len(grids)) + + self._i = 0 + self._p = 0 + + self._counter = 0 + self._best_estimator = None + self._bandits = self._create_bandits(estimator, nr_estimators) + + # Meta-programming + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" + + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return int(val) + elif hp_type == float: + val = self._sample[self._i][self._p] * (hp_range[1] - hp_range[0]) + hp_range[0] + self._p += 1 + return val + + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ + + # Sub-component needs to be instantiated + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 + + if operation == "combine": + est_2 = est_2[1] + else: + est_2 = {} + + sub_config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name + else: + sub_hp_prefix = None + + sub_config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], + func=func, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(est_1, numbers.Number): + if operation == "generate": + return self.__generate(hp_data) + if operation == "scale": + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) + return + + # The sub-parameters need to be expanded + config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] + + if operation == "combine": + sub_est_2 = est_2[sub_hp_name] + else: + sub_est_2 = {} + + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name + else: + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + + return config + + def _random_config(self): + return self._traverse_hps( + operation="generate", + hp_data=self.params_range, + est_1=self.estimator._get_params(), + ) + + def _create_bandits(self, model, nr_estimators) -> typing.List: + bandits = [None] * nr_estimators + for i in range(nr_estimators): + bandits[i] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True), + ) + self._i += 1 + self._p = 0 + self._i = 0 + return bandits + + def _sort_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._bandits.sort(key=lambda mw: mw.metric.get()) + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._traverse_hps( + operation="scale", + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled + ) + return scaled + + @property + def _models_converged(self) -> bool: + if len(self._bandits) == 3: + return True + else: + return False + + def _learn_not_converged(self, x, y): + for wrap in self._bandits: + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) + wrap.metric.update(y, y_pred) + wrap.estimator.learn_one(x, y) + + # Keep the simplex ordered + + + if self._n == self.grace_period: + self._sort_bandits() + self._n = 0 + + + def learn_one(self, x, y): + self._n += 1 + self._counter += 1 + + self._learn_not_converged(x, y) + + return self + + @property + def best(self): + if not self._converged: + # Lazy selection of the best model + self._sort_bandits() + return self._bandits[0].estimator + + return self._best_estimator + + @property + def converged(self): + return self._converged + + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) + + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) + + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) + + def debug_one(self, x, **kwargs): + try: + return self.best.debug_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) diff --git a/river_extra/model_selection/messpt.py b/river_extra/model_selection/messpt.py new file mode 100644 index 0000000..b821f3b --- /dev/null +++ b/river_extra/model_selection/messpt.py @@ -0,0 +1,863 @@ +from ast import operator +import collections +import copy +import math +import numbers +import random +import typing + +import numpy as np +import time + +from river import anomaly, base, compose, drift, metrics, utils + + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + + +class MICRODE(base.Estimator): + """MESSPT + + Using a Micro-evolutionary based on Differential Evolution for Self Parameter Tuning + + + """ + + _START_RANDOM = "random" + _START_WARM = "warm" + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + seed: int = None, + n_ind: int = 4, + reset_one: int = 0, + len_ind: int = 3, + gen_to_conv: int = 5, + F_ini: float = 0.5, + CR_ini: float= 0.5, + aug: float = 0.025, + num_reset: int=1, + + ): + super().__init__() + + #To prevent concept drift before convergence, always a random option is included in each run. + #If random option is the best after gp_init_rand grace periods, reset + #If reset_one is 0, not to include this random option in the algorithm. Else yes + + self.random_option = None + self._old_b_rand_opt = None + self.reset_one = reset_one + self._gp_init_rand = num_reset + self._random_best = False + #If rand is best is 0, currently the random option is not the best. 1 else + self._rand_is_best = 0 + + ##To measure cost + ## + self._time_to_conv = [] + self._time_acc = 0 + self._num_ex = 0 + self._num_ex_array = [] + self._num_models = 0 + self._num_models_array = [] + ## + + self.len_ind = len_ind + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + self._converged = False + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + #self.first_pop = True + self.seed = seed + self._rng = random.Random(self.seed) + self._n = 0 + #To check differences between current and old best + self._old_b = None + #number of current generations with same (or low differences) individual as best + self._gen_same_b = 0 + #number of generations (max) to consider convergence + self._gen_to_conv = gen_to_conv + #number of individuals into pop + self.n_ind = n_ind + #probability of mutation + self.current_pop = [] + + self._best_i = None + self._best_child = False + + self.current_children = [] + self._best_estimator = None + + #F and CR values control mutation and cross. If is_adaptive is True, they will change dynamically. + #Else, they will be static values during all the process + + self.is_adaptive = True + #How F and CR changes after each grace period + self.aug = aug + + #initial values of F and CR + self.F_ini = F_ini + self.CR_ini = CR_ini + self.F = self.F_ini + self.CR = self.CR_ini + + #Create first population + self._create_pop(self.n_ind) + + #Generate new gen. It could be int, float or discrete. Discrete option is not tested + def __generate(self, p_data): + p_type, p_range = p_data + if p_type == int: + return self._rng.randint(p_range[0], p_range[1]) + elif p_type == float: + return self._rng.uniform(p_range[0], p_range[1]) + #DISCRET (Not tested) + ''' + elif p_type == "discrete": + choice = self._rng.randint(0, len(p_range)) + return p_range[choice] + ''' + + #Generate a new random configuration + + def _random_config(self): + return self._recurse_params( + operation = "generate", + p_data=self.params_range, + e1_data=self.estimator._get_params() + ) + + #Create a new wrapper (individual + metric) + + def _create_wrapper(self, ind, metric, est): + w = ModelWrapper(est.clone(ind, include_attributes=True), metric) + return w + + + #Create a new pop o n individuals from scratch + + def _create_pop(self, n): + self.current_pop = [None]*n + for i in range(n): + self.current_pop[i] = self._create_wrapper(self._random_config(), + self.metric.clone(include_attributes=True), self.estimator) + + #Scale for checking similarities between best individuals (old and current) + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._recurse_params( + operation="scale", + p_data=self.params_range, + e1_data=orig, + prefix="", + scaled=scaled + ) + return scaled + + @property + def best(self): + + return self._best_estimator + + @property + def random_best(self): + return self._random_best + + + + @property + + #If best and old best are too similar, we consider they are equal in this generations. + #If they are equal (low differences) after gen_to_conv generations, we consider convergence + + def _models_converged(self) -> bool: + # Normalize params to ensure they contribute equally to the stopping criterion + + scaled_params_b = self._normalize_flattened_hyperspace( + self._best_estimator._get_params() + ) + r_sphere=1 + + if self._old_b != None: + r_sphere = utils.math.minkowski_distance(scaled_params_b, + self._old_b, p=2) + + #If low differences: + + if r_sphere < self.convergence_sphere: + #Number of generations in which they are similar is the maximum (convergence) + if self._gen_same_b == self._gen_to_conv: + self._gen_same_b = 0 + self._old_b = None + return True + else: + #one more generation in which current and old are similar + self._gen_same_b = self._gen_same_b+1 + return False + #Consider differences (0 generations without similarity) + else: + self._gen_same_b = 0 + self._old_b = scaled_params_b + return False + + + #Combine parameters from 3 individuals + + def __combine(self, p_info, param1, param2, param3, func): + + p_type, p_range = p_info + new_val = func(param1, param2, param3) + + # Range sanity checks + if new_val < p_range[0]: + new_val = p_range[0] + if new_val > p_range[1]: + new_val = p_range[1] + + new_val = round(new_val, 0) if p_type == int else new_val + return new_val + + + #Combine 3 estimators to generate one new. new = best + F(r1-r2) + def _gen_new_estimator(self, e1, e2, e3, func): + """Generate new configuration given two estimators and a combination function.""" + e1_p = e1.estimator._get_params() + e2_p = e2.estimator._get_params() + e3_p = e3.estimator._get_params() + + + new_config = self._recurse_params( + operation="combine", + p_data=self.params_range, + e1_data=e1_p, + func=func, + e2_data=e2_p, + e3_data=e3_p, + ) + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(self._best_estimator), + self.metric.clone(include_attributes=True), + + ) + + new.estimator.mutate(new_config) + + return new + + def __combine_cross_rate(self, p_info, param1, param2, num, index_change, gen, func): + + p_type, p_range = p_info + new_val = func(param1, param2, num, index_change, gen) + + # Range sanity checks + if new_val < p_range[0]: + new_val = p_range[0] + if new_val > p_range[1]: + new_val = p_range[1] + + new_val = round(new_val, 0) if p_type == int else new_val + return new_val + + def _gen_new_estimator_cross_rate(self, e1, e2, index_change, func): + """Generate new configuration given two estimators and a combination function.""" + + e1_p = e1.estimator._get_params() + e2_p = e2.estimator._get_params() + + + new_config = self._recurse_params( + operation="combine_cr", + p_data=self.params_range, + e1_data=e1_p, + index_change=index_change, + func=func, + e2_data=e2_p, + gen=0 + ) + # Modify the current best contender with the new hyperparameter values + new = ModelWrapper( + copy.deepcopy(e2.estimator), + self.metric.clone(include_attributes=True), + + ) + new.estimator.mutate(new_config) + + return new + + + def __flatten(self, prefix, scaled, p_info, e_info): + _, p_range = p_info + interval = p_range[1] - p_range[0] + scaled[prefix] = (e_info - p_range[0]) / interval + + + #Generate: Generate a new random individual + #Scale: Scale for checking similarities + #Combine: Combine 3 individuals + #Combine_cr: Cross two individuals: the current one and the obtained from combinated (mutation) 3 individuals. + #To combine_cr: gen by gen check if include the gen from the current individual or from the mutation. + + + def _recurse_params( + self, operation, p_data, e1_data, *, index_change=None, + func=None, e2_data=None, e3_data=None, + prefix=None, scaled=None, gen=None + ): + + # Sub-component needs to be instantiated + if isinstance(e1_data, tuple): + sub_class, sub_data1 = e1_data + + if operation=="combine_cr": + _, sub_data2 = e2_data + sub_data3 = {} + + elif operation == "combine": + _, sub_data2 = e2_data + _, sub_data3 = e3_data + + else: + sub_data2 = {} + sub_data3 = {} + + + sub_config = {} + for sub_param, sub_info in p_data.items(): + if operation == "scale": + sub_prefix = prefix + "__" + sub_param + else: + sub_prefix = None + sub_config[sub_param] = self._recurse_params( + operation=operation, + p_data=sub_info, + e1_data=sub_data1[sub_param], + func=func, + e2_data=sub_data2.get(sub_param, None), + e3_data=sub_data3.get(sub_param, None), + index_change=index_change, + prefix=sub_prefix, + scaled=scaled, + gen=gen + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(p_data, tuple): + if operation == "generate": + return self.__generate(p_data) + if operation == "scale": + self.__flatten(prefix, scaled, p_data, e1_data) + return + if operation == "combine_cr": + num = self._rng.uniform(0,1) + res = self.__combine_cross_rate(p_data, e1_data, e2_data, num, + index_change, gen, func) + return res + + # combine + #NEW + p_type, p_range = p_data + if p_type == int or p_type == float: + return self.__combine(p_data, e1_data, e2_data, e3_data, func) + else: + return self.__combine(p_data, e1_data, e2_data, e3_data, func)# TODO: func_disc + # The sub-parameters need to be expanded + config = {} + for p_name, p_info in p_data.items(): + e1_info = e1_data[p_name] + + if operation == "combine": + e2_info = e2_data[p_name] + e3_info = e3_data[p_name] + + elif operation == "combine_cr": + e2_info = e2_data[p_name] + e3_info = {} + + else: + e2_info = {} + e3_info = {} + + if operation == "scale": + sub_prefix = prefix + "__" + p_name if len(prefix) > 0 else p_name + else: + sub_prefix = None + + if not isinstance(p_info, dict): + config[p_name] = self._recurse_params( + operation=operation, + p_data=p_info, + e1_data=e1_info, + func=func, + e2_data=e2_info, + e3_data=e3_info, + index_change=index_change, + prefix=sub_prefix, + scaled=scaled, + gen=gen + ) + else: + sub_config = {} + for sub_name, sub_info in p_info.items(): + if operation == "scale": + sub_prefix2 = sub_prefix + "__" + sub_name + else: + sub_prefix2 = None + sub_config[sub_name] = self._recurse_params( + operation=operation, + p_data=sub_info, + e1_data=e1_info[sub_name], + func=func, + e2_data=e2_info.get(sub_name, None), + e3_data=e3_info.get(sub_name, None), + index_change=index_change, + prefix=sub_prefix2, + scaled=scaled, + gen=gen + ) + if operation == "combine_cr": + gen = gen+1 + config[p_name] = sub_config + return config + + + #Combine for discrete. It is not tested! + + ''' + def func_disc(self, v1, v2, v3): + r_f_c = self._rng.uniform(0,1) + to_c = v2 if r_f_c <= 0.5 else v3 + r_f_c_2 = self._rng.uniform(0,1) + res = to_c if r_f_c_2 < self.F/(1+self.F) else v1 + return res + ''' + + #Cross operator from differential evolution. Best_1 operator: new = best + F(r1-r2) + + + def _de_cross_best_1(self, i): + #Select 2 individuals from current list. It has to be different from the current (i) individual + list_el = np.ndarray.tolist(np.arange(1,self.n_ind)) + if i!=0: + list_el.remove(i) + r1, r2 = self._rng.sample(list_el, 2) + #best in 0 pos + n_p = self._gen_new_estimator( + self.current_pop[0], self.current_pop[r1], self.current_pop[r2], + lambda h1, h2, h3: h1 + self.F*(h2-h3) + ) + return n_p + + def _ev_op_crossover( + self, operation, ind=None, index=None): + + return self._de_cross_best_1(index) + + + #To end cross. Check if new gen will be obtained from current individual or mutation + + def func_cr(self, p1, p2, num, index_change, gen): + + if num We need to start the optimization process from scratch + if self.drift_detector.drift_detected: + self.F = self.F_ini + self.CR = self.CR_ini + # if self.reset_one==1: + # print("drift detected mder") + # else: + # print("drift detected mde") + + self._n = 0 + self._converged = False + self._best_child = False + + #Except for the current best (i=0), all the others are restarted from scratch + + for i in range(1, self.n_ind): + self.current_pop[i] = self._create_wrapper(self._random_config(), + #self._best_i.metric.clone(include_attributes=True) + self.metric.clone(include_attributes=True), self.estimator + + ) + + #Current best is kept in the population restarted + self.current_pop[0] = ModelWrapper(copy.deepcopy(self._best_estimator), + self.metric.clone(include_attributes=True)) + + + + + # There is no proven best model right now + self._best_i = None + self._best_estimator = None + + #Variables related to random option + self.random_option = None + self._random_best = None + + #Variables related to cost + self._time_acc = 0 + self._num_ex = 0 + self._num_models = 0 + + return + + + #If concept drift not detected + self._best_estimator.learn_one(x, y) + + + #Learn one. If converged, call to _learn converged. Else: + ''' + Else: Update and learn with each individual for this data. If we reach a grace period (number) + of evaluated = grace_period -> Run micro evolutionary algorithm + + ''' + def learn_one(self, x, y): + + if self._converged ==True: + self._n = self._n + 1 + self._learn_converged(x, y) + else: + #num ex not converged + t1 = time.time() + self._num_ex = self._num_ex + 1 + + #update and learn individuals of pop + for wrap in self.current_pop: + self._update(wrap, x, y) + + self._learn_one(wrap, x, y) + + self._sort_c_pop() + + #if current children exists: update and learn for each individual + if self.current_children != []: + for wrap in self.current_children: + self._update(wrap, x, y) + self._learn_one(wrap, x, y) + self._sort_c_children() + + #select the best between current pop, current children pop and the random option + ####### + #### + if self.current_children != []: + if self.metric.bigger_is_better: + if self.current_pop[0].metric.get() < self.current_children[0].metric.get(): + self._best_i = self.current_children[0] + self._best_child = True + else: + self._best_i = self.current_pop[0] + self._best_child = False + else: + if self.current_pop[0].metric.get() > self.current_children[0].metric.get(): + self._best_i = self.current_children[0] + self._best_child = True + else: + self._best_i = self.current_pop[0] + self._best_child = False + else: + self._best_i = self.current_pop[0] + self._best_child = False + + + self._random_best = False + if self.random_option != None: + #update and learn for the random option + self._update(self.random_option, x, y) + self._learn_one(self.random_option, x, y) + #check if random is the best + if self.metric.bigger_is_better: + + if self.random_option.metric.get() > self._best_i.metric.get(): + + + self._random_best = True + else: + if self.random_option.metric.get() < self._best_i.metric.get(): + + + + self._random_best = True + #### + ####### + + + self._best_estimator = self._best_i.estimator + + self._n = self._n + 1 + #if reach grace period. Run micro-ev algorithm + if self._n % self.grace_period == 0: + #If random option is the best + if self._random_best == True: + #restart values for micro-ev algorithm to look for better options. + self._num_models = self._num_models + self.n_ind*2+1 + self.F = self.F_ini + self.CR = self.CR_ini + self._rand_is_best = self._rand_is_best+1 + #If the number of grace periods (in a row) in which random is the best is > than + # gp_init_rand --> Restart + if self._rand_is_best >= self._gp_init_rand: + + for i in range(2, self.n_ind): + self.current_pop[i] = self._create_wrapper(self._random_config(), + self.metric.clone(include_attributes=True), self.estimator + + ) + + self.current_pop[0] = ModelWrapper(copy.deepcopy(self.random_option.estimator), + self.metric.clone(include_attributes=True)) + + #copy best before rand + self.current_pop[1] = ModelWrapper(copy.deepcopy(self._best_i.estimator), + self.metric.clone(include_attributes=True)) + + self._old_b_rand_opt = None + self.current_children = [] + self.F = self.F_ini + self.CR = self.CR_ini + self._n = 0 + self._converged = False + self._best_child = False + self._random_best = False + self.random_option = None + + # There is no proven best model right now + self._best_i = None + self._best_estimator = None + self._rand_is_best = 0 + + #If not reached the gp_init_rand number: run the micro-ev algorithm + + else: + + if self.is_adaptive: + #to fast conv + self.F = self.F - self.aug + self.CR = self.CR + self.aug + if self.F < 0.0: + self.F = 0.0 + if self.CR > 1: + self.CR = 1 + + self._generate_next_current_pop() + self.current_children=[] + self._best_child = False + + #Check convergence + + if self._models_converged: + t2 = time.time() + # if self.reset_one==0: + # print("mde convergence") + # else: + # print("mder convergence") + self._time_acc = self._time_acc + (t2-t1) + self._time_to_conv.append(self._time_acc) + self._num_ex_array.append(self._num_ex) + self._num_models_array.append(self._num_models) + self._converged = True + else: + self._cross_current_pop() + + + + #We are in the first grace period. Just cross the current pop, not create a new one + elif self._n / self.grace_period == 1: + self._num_models = self._num_models + self.n_ind + self._rand_is_best = 0 + + #firstly, models are mixed + self._cross_current_pop() + + for j in range(self.n_ind): + self.current_pop[j] = ModelWrapper( + copy.deepcopy(self.current_pop[j].estimator), + self.metric.clone(include_attributes=True), + + ) + + #Most ordinary case: Random is not the best and not the first generation: Run the micro_ev + #algorithm + + else: + self._rand_is_best = 0 + self._num_models = self._num_models + self.n_ind*2+1 + + if self.is_adaptive: + #to fast conv + self.F = self.F - self.aug + self.CR = self.CR + self.aug + if self.F < 0.0: + self.F = 0.0 + if self.CR > 1: + self.CR = 1 + + self._generate_next_current_pop() + self.current_children=[] + self._best_child = False + + if self._models_converged: + t2 = time.time() + # if self.reset_one==0: + # print("mde convergence") + # else: + # print("mder convergence") + self._time_acc = self._time_acc + (t2-t1) + self._time_to_conv.append(self._time_acc) + self._num_ex_array.append(self._num_ex) + self._num_models_array.append(self._num_models) + self._converged = True + else: + self._cross_current_pop() + + + + t2 = time.time() + self._time_acc = self._time_acc + (t2-t1) + return self + + + def predict_one(self, x, **kwargs): + + if self._random_best: + return self.random_option.estimator.predict_one(x, **kwargs) + + + elif self._best_estimator == None: + self._sort_c_pop() + return self.current_pop[0].estimator.predict_one(x, **kwargs) + + else: + return self._best_estimator.predict_one(x, **kwargs) + + + @property + def converged(self): + return self._converged + + + #To measure cost + + @property + def time_to_conv(self): + if self.converged: + return self._time_to_conv + else: + self._time_to_conv.append(self._time_acc) + return self._time_to_conv + + @property + def num_ex(self): + if self.converged: + return self._num_ex_array + else: + self._num_ex_array.append(self._num_ex) + return self._num_ex_array + + @property + def num_models(self): + if self.converged: + return self._num_models_array + else: + self._num_models_array.append(self._num_models) + return self._num_models_array \ No newline at end of file diff --git a/river_extra/model_selection/random_search.py b/river_extra/model_selection/random_search.py new file mode 100644 index 0000000..38bcd15 --- /dev/null +++ b/river_extra/model_selection/random_search.py @@ -0,0 +1,296 @@ +import statistics +from ast import operator +import collections +import copy +import math +import numbers +import random +import typing +import pandas as pd +import numpy as np +from itertools import combinations + +from scipy.stats import qmc +from tqdm import tqdm + +import numpy as np + +# TODO use lazy imports where needed +from river import anomaly, base, compose, drift, metrics, utils, preprocessing, tree + +ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + + +class Random_Search(base.Estimator): + + def __init__( + self, + estimator: base.Estimator, + metric: metrics.base.Metric, + params_range: typing.Dict[str, typing.Tuple], + drift_input: typing.Callable[[float, float], float], + grace_period: int = 500, + drift_detector: base.DriftDetector = drift.ADWIN(), + convergence_sphere: float = 0.001, + nr_estimators: int = 50, + seed: int = None, + ): + super().__init__() + self.estimator = estimator + self.metric = metric + self.params_range = params_range + self.drift_input = drift_input + + self.grace_period = grace_period + self.drift_detector = drift_detector + self.convergence_sphere = convergence_sphere + + self.nr_estimators = nr_estimators + + self.seed = seed + + self._n = 0 + self._converged = False + self._rng = random.Random(self.seed) + + self._counter=0 + self._best_estimator = None + self._bandits = self._create_bandits(estimator,nr_estimators) + + # Meta-programming + border = self.estimator + if isinstance(border, compose.Pipeline): + border = border[-1] + + if isinstance(border, (base.Classifier, base.Regressor)): + self._scorer_name = "predict_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "score_one" + elif isinstance(border, anomaly.base.AnomalyDetector): + self._scorer_name = "classify" + + def __generate(self, hp_data) -> numbers.Number: + hp_type, hp_range = hp_data + if hp_type == int: + return self._rng.randint(hp_range[0], hp_range[1]) + elif hp_type == float: + return self._rng.uniform(hp_range[0], hp_range[1]) + + def __flatten(self, prefix, scaled_hps, hp_data, est_data): + hp_range = hp_data[1] + interval = hp_range[1] - hp_range[0] + scaled_hps[prefix] = (est_data - hp_range[0]) / interval + + def _traverse_hps( + self, operation: str, hp_data: dict, est_1, *, func=None, est_2=None, hp_prefix=None, scaled_hps=None + ) -> typing.Optional[typing.Union[dict, numbers.Number]]: + """Traverse the hyperparameters of the estimator/pipeline and perform an operation. + + Parameters + ---------- + operation + The operation that is intented to apply over the hyperparameters. Can be either: + "combine" (combine parameters from two pipelines), "scale" (scale a flattened + version of the hyperparameter hierarchy to use in the stopping criteria), or + "generate" (create a new hyperparameter set candidate). + hp_data + The hyperparameter data which was passed by the user. Defines the ranges to + explore for each hyperparameter. + est_1 + The hyperparameter structure of the first pipeline/estimator. Such structure is obtained + via a `_get_params()` method call. Both 'hp_data' and 'est_1' will be jointly traversed. + func + A function that is used to combine the values in `est_1` and `est_2`, in case + `operation="combine"`. + est_2 + A second pipeline/estimator which is going to be combined with `est_1`, in case + `operation="combine"`. + hp_prefix + A hyperparameter prefix which is used to identify each hyperparameter in the hyperparameter + hierarchy when `operation="scale"`. The recursive traversal will modify this prefix accordingly + to the current position in the hierarchy. Initially it is set to `None`. + scaled_hps + Flattened version of the hyperparameter hierarchy which is used for evaluating stopping criteria. + Set to `None` and defined automatically when `operation="scale"`. + """ + + # Sub-component needs to be instantiated + if isinstance(est_1, tuple): + sub_class, est_1 = est_1 + + if operation == "combine": + est_2 = est_2[1] + else: + est_2 = {} + + sub_config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name + else: + sub_hp_prefix = None + + sub_config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=est_1[sub_hp_name], + func=func, + est_2=est_2.get(sub_hp_name, None), + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + return sub_class(**sub_config) + + # We reached the numeric parameters + if isinstance(est_1, numbers.Number): + if operation == "generate": + return self.__generate(hp_data) + if operation == "scale": + self.__flatten(hp_prefix, scaled_hps, hp_data, est_1) + return + + + # The sub-parameters need to be expanded + config = {} + for sub_hp_name, sub_hp_data in hp_data.items(): + sub_est_1 = est_1[sub_hp_name] + + if operation == "combine": + sub_est_2 = est_2[sub_hp_name] + else: + sub_est_2 = {} + + if operation == "scale": + sub_hp_prefix = hp_prefix + "__" + sub_hp_name if len(hp_prefix) > 0 else sub_hp_name + else: + sub_hp_prefix = None + + config[sub_hp_name] = self._traverse_hps( + operation=operation, + hp_data=sub_hp_data, + est_1=sub_est_1, + func=func, + est_2=sub_est_2, + hp_prefix=sub_hp_prefix, + scaled_hps=scaled_hps, + ) + + return config + + def _random_config(self): + return self._traverse_hps( + operation="generate", + hp_data=self.params_range, + est_1=self.estimator._get_params(), + ) + + def _create_bandits(self, model, nr_estimators) -> typing.List: + bandits = [None] * nr_estimators + for i in range(nr_estimators): + bandits[i] = ModelWrapper( + model.clone(self._random_config(), include_attributes=True), + self.metric.clone(include_attributes=True), + ) + return bandits + + def _sort_bandits(self): + """Ensure the simplex models are ordered by predictive performance.""" + if self.metric.bigger_is_better: + self._bandits.sort(key=lambda mw: mw.metric.get(), reverse=True) + else: + self._bandits.sort(key=lambda mw: mw.metric.get()) + + def _normalize_flattened_hyperspace(self, orig): + scaled = {} + self._traverse_hps( + operation="scale", + hp_data=self.params_range, + est_1=orig, + hp_prefix="", + scaled_hps=scaled + ) + return scaled + + @property + def _models_converged(self) -> bool: + if len(self._bandits)==3: + return True + else: + return False + + def _learn_not_converged(self, x, y): + for wrap in self._bandits: + scorer = getattr(wrap.estimator, self._scorer_name) + y_pred = scorer(x) + wrap.metric.update(y, y_pred) + wrap.estimator.learn_one(x, y) + + # Keep the simplex ordered + + + if self._n == self.grace_period: + self._sort_bandits() + self._n = 0 + + + def learn_one(self, x, y): + self._n += 1 + self._counter += 1 + + self._learn_not_converged(x, y) + + return self + + @property + def best(self): + if not self._converged: + # Lazy selection of the best model + self._sort_bandits() + return self._bandits[0].estimator + + return self._best_estimator + + @property + def converged(self): + return self._converged + + def predict_one(self, x, **kwargs): + try: + return self.best.predict_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_one' is not supported in {border.__class__.__name__}." + ) + + def predict_proba_one(self, x, **kwargs): + try: + return self.best.predict_proba_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'predict_proba_one' is not supported in {border.__class__.__name__}." + ) + + def score_one(self, x, **kwargs): + try: + return self.best.score_one(x, **kwargs) + except NotImplementedError: + border = self.best + if isinstance(border, compose.Pipeline): + border = border[-1] + raise AttributeError( + f"'score_one' is not supported in {border.__class__.__name__}." + ) + + def debug_one(self, x, **kwargs): + try: + return self.best.debug_one(x, **kwargs) + except NotImplementedError: + raise AttributeError( + f"'debug_one' is not supported in {self.best.__class__.__name__}." + ) diff --git a/river_extra/model_selection/sspt.py b/river_extra/model_selection/sspt.py index 2058af7..1498c95 100644 --- a/river_extra/model_selection/sspt.py +++ b/river_extra/model_selection/sspt.py @@ -12,6 +12,7 @@ ModelWrapper = collections.namedtuple("ModelWrapper", "estimator metric") + class SSPT(base.Estimator): """Single-pass Self Parameter Tuning @@ -84,7 +85,6 @@ def __init__( elif isinstance(border, anomaly.base.AnomalyDetector): self._scorer_name = "classify" - def __generate(self, hp_data) -> numbers.Number: hp_type, hp_range = hp_data if hp_type == int: @@ -435,7 +435,7 @@ def _learn_not_converged(self, x, y): scaled_params_w = self._normalize_flattened_hyperspace( self._simplex[2].estimator._get_params(), ) - print("----------") + print("---SPT---") print( "B:", list(scaled_params_b.values()), "Score:", self._simplex[0].metric )