From 4490aa298a26693dd224e95e18f955d26045395a Mon Sep 17 00:00:00 2001 From: Kobi Felton Date: Mon, 13 Feb 2023 23:11:01 +0000 Subject: [PATCH] TSEMO fixes (#234) * Change TSEMO defaults, fix tsemo maximization, add test cases * Fixes to improve tests * Update defaults * Update to least_duration algorithm * Bump up number of iterations threshold * Fix tsemo again --- summit/benchmarks/test_functions.py | 12 ++++++++---- summit/strategies/tsemo.py | 23 +++++++++++++---------- summit/tests/test_runner.py | 4 ++-- summit/tests/test_strategies.py | 9 +++++---- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/summit/benchmarks/test_functions.py b/summit/benchmarks/test_functions.py index cc659b1c..ab583a76 100644 --- a/summit/benchmarks/test_functions.py +++ b/summit/benchmarks/test_functions.py @@ -650,13 +650,14 @@ def _run(self, conditions, **kwargs): class VLMOP2(Experiment): - def __init__(self, **kwargs): - domain = self._setup_domain(2, 2) + def __init__(self, maximize: bool = False, **kwargs): + domain = self._setup_domain(2, 2, maximize) self.nvars = 2 self.nobjs = 2 + self.maximize= maximize super().__init__(domain) - def _setup_domain(self, nobjs, nvars): + def _setup_domain(self, nobjs, nvars, maximize): variables = [ ContinuousVariable(f"x_{i}", f"Decision variable {i}", bounds=[-2, 2]) for i in range(nvars) @@ -667,7 +668,7 @@ def _setup_domain(self, nobjs, nvars): f"Objective {i}", bounds=[-2, 2], is_objective=True, - maximize=False, + maximize=maximize, ) for i in range(nobjs) ] @@ -686,6 +687,9 @@ def _run(self, conditions, **kwargs): y2 = 1 - np.exp(-1 * part2) f = np.hstack((y1, y2)) + if self.maximize: + f *= -1.0 + # Convert to dataset for i in range(self.nobjs): conditions[(f"y_{i}", "DATA")] = f[:, i][0] diff --git a/summit/strategies/tsemo.py b/summit/strategies/tsemo.py index 378e0369..cf3d90ac 100644 --- a/summit/strategies/tsemo.py +++ b/summit/strategies/tsemo.py @@ -106,8 +106,6 @@ class TSEMO(Strategy): """ def __init__(self, domain, transform=None, **kwargs): - from GPy.kern import Exponential - Strategy.__init__(self, domain, transform, **kwargs) # Categorical variable options @@ -138,7 +136,7 @@ def __init__(self, domain, transform=None, **kwargs): self.n_retries = kwargs.get("n_retries", 10) # NSGA-II tsemo_settings - self.generations = kwargs.get("generations", 100) + self.generations = kwargs.get("generations", 1000) self.pop_size = kwargs.get("pop_size", 100) self.logger = kwargs.get("logger", logging.getLogger(__name__)) @@ -202,12 +200,12 @@ def suggest_experiments(self, num_experiments, prev_res: DataSet = None, **kwarg # Train and sample n_outputs = len(self.domain.output_variables) train_results = [0] * n_outputs - models = [0] * n_outputs + self.models = [0] * n_outputs rmse_train_spectral = np.zeros(n_outputs) for i, v in enumerate(self.domain.output_variables): # Training - models[i] = ThompsonSampledModel(v.name) - train_results[i] = models[i].fit( + self.models[i] = ThompsonSampledModel(v.name) + train_results[i] = self.models[i].fit( inputs, outputs[[v.name]], n_retries=self.n_retries, @@ -215,7 +213,7 @@ def suggest_experiments(self, num_experiments, prev_res: DataSet = None, **kwarg ) # Evaluate spectral sampled functions - sample_f = lambda x: np.atleast_2d(models[i].rff(x)).T + sample_f = lambda x: np.atleast_2d(self.models[i].rff(x)).T rmse_train_spectral[i] = rmse( sample_f(inputs.to_numpy().astype("float")), outputs[[v.name]].to_numpy().astype("float"), @@ -233,13 +231,13 @@ def suggest_experiments(self, num_experiments, prev_res: DataSet = None, **kwarg if (self.domain.num_continuous_dimensions() == 0) and ( self.domain.num_categorical_variables() == 1 ): - X, yhat = self._categorical_enumerate(models) + X, yhat = self._categorical_enumerate(self.models) # Mixed domains elif self.categorical_combos is not None and len(self.input_columns) > 1: - X, yhat = self._nsga_optimize_mixed(models) + X, yhat = self._nsga_optimize_mixed(self.models) # Continous domains elif self.categorical_combos is None and len(self.input_columns) > 0: - X, yhat = self._nsga_optimize(models) + X, yhat = self._nsga_optimize(self.models) # Return if no suggestiosn found if X.shape[0] == 0 and yhat.shape[0] == 0: @@ -329,6 +327,9 @@ def _nsga_optimize_mixed(self, models): y = np.atleast_2d(self.internal_res.F).tolist() X = DataSet(X, columns=problem.X_columns) y = DataSet(y, columns=[v.name for v in self.domain.output_variables]) + for v in self.domain.output_variables: + if v.maximize: + y[v.name] = -y[v.name] # Add in categorical variables for key, value in combo.to_dict().items(): X[key] = value @@ -581,6 +582,7 @@ def predict(self, X: DataSet, **kwargs): return self.rff(X) def save(self, filepath=None): + import pyrff if filepath is None: filepath = get_summit_config_path() / "tsemo" / str(self.uuid_val) os.makedirs(filepath, exist_ok=True) @@ -588,6 +590,7 @@ def save(self, filepath=None): pyrff.save_rffs([self.rff], filepath) def load(self, filepath=None): + import pyrff if filepath is None: filepath = get_summit_config_path() / "tsemo" / str(self.uuid_val) os.makedirs(filepath, exist_ok=True) diff --git a/summit/tests/test_runner.py b/summit/tests/test_runner.py index c18d48f1..5b832331 100644 --- a/summit/tests/test_runner.py +++ b/summit/tests/test_runner.py @@ -140,7 +140,7 @@ def test_runner_mo_integration(strategy, experiment): SOBO, TSEMO, ]: - # only run on strategies that work with categorical variables deireclty + # only run on strategies that work with categorical variables direclty return elif strategy == TSEMO: s = strategy(exp.domain) @@ -154,7 +154,7 @@ def test_runner_mo_integration(strategy, experiment): s = strategy(exp.domain, transform=transform) iterations = 3 - r = Runner(strategy=s, experiment=exp, max_iterations=iterations, batch_size=1) + r = Runner(strategy=s, experiment=exp, num_initial_experiments=8, max_iterations=iterations, batch_size=1) r.run() # Try saving and loading diff --git a/summit/tests/test_strategies.py b/summit/tests/test_strategies.py index a691e9d3..dcc1edd3 100644 --- a/summit/tests/test_strategies.py +++ b/summit/tests/test_strategies.py @@ -6,6 +6,7 @@ from fastprogress.fastprogress import progress_bar import numpy as np import os +import matplotlib.pyplot as plt def test_strategy(): class MockStrategy(Strategy): @@ -692,10 +693,10 @@ def test_mtbo( @pytest.mark.parametrize("batch_size", [1, 2, 10]) -def test_tsemo(batch_size, test_num_improve_iter=2, save=False): +@pytest.mark.parametrize("maximize", [True, False]) +def test_tsemo(batch_size, maximize, test_num_improve_iter=5, save=False, plot=True): num_inputs = 2 - num_objectives = 2 - lab = VLMOP2() + lab = VLMOP2(maximize=maximize) strategy = TSEMO(lab.domain) experiments = strategy.suggest_experiments(5 * num_inputs) @@ -728,7 +729,7 @@ def test_tsemo(batch_size, test_num_improve_iter=2, save=False): ) ) break - # assert hv > 117.0 + assert hv > 110.0 @pytest.mark.parametrize(