TSEMO fixes (#234)

* Change TSEMO defaults, fix tsemo maximization, add test cases

* Fixes to improve tests

* Update defaults

* Update to least_duration algorithm

* Bump up number of iterations threshold

* Fix tsemo again

summit/benchmarks/test_functions.py

@@ -650,13 +650,14 @@ def _run(self, conditions, **kwargs):
 
 
 class VLMOP2(Experiment):
-    def __init__(self, **kwargs):
-        domain = self._setup_domain(2, 2)
+    def __init__(self, maximize: bool = False, **kwargs):
+        domain = self._setup_domain(2, 2, maximize)
         self.nvars = 2
         self.nobjs = 2
+        self.maximize= maximize
         super().__init__(domain)
 
-    def _setup_domain(self, nobjs, nvars):
+    def _setup_domain(self, nobjs, nvars, maximize):
         variables = [
             ContinuousVariable(f"x_{i}", f"Decision variable {i}", bounds=[-2, 2])
             for i in range(nvars)
@@ -667,7 +668,7 @@ def _setup_domain(self, nobjs, nvars):
                 f"Objective {i}",
                 bounds=[-2, 2],
                 is_objective=True,
-                maximize=False,
+                maximize=maximize,
             )
             for i in range(nobjs)
         ]
@@ -686,6 +687,9 @@ def _run(self, conditions, **kwargs):
         y2 = 1 - np.exp(-1 * part2)
         f = np.hstack((y1, y2))
 
+        if self.maximize:
+            f *= -1.0
+
         # Convert to dataset
         for i in range(self.nobjs):
             conditions[(f"y_{i}", "DATA")] = f[:, i][0]

summit/strategies/tsemo.py

@@ -106,8 +106,6 @@ class TSEMO(Strategy):
     """
 
     def __init__(self, domain, transform=None, **kwargs):
-        from GPy.kern import Exponential
-
         Strategy.__init__(self, domain, transform, **kwargs)
 
         # Categorical variable options
@@ -138,7 +136,7 @@ def __init__(self, domain, transform=None, **kwargs):
         self.n_retries = kwargs.get("n_retries", 10)
 
         # NSGA-II tsemo_settings
-        self.generations = kwargs.get("generations", 100)
+        self.generations = kwargs.get("generations", 1000)
         self.pop_size = kwargs.get("pop_size", 100)
 
         self.logger = kwargs.get("logger", logging.getLogger(__name__))
@@ -202,20 +200,20 @@ def suggest_experiments(self, num_experiments, prev_res: DataSet = None, **kwarg
         # Train and sample
         n_outputs = len(self.domain.output_variables)
         train_results = [0] * n_outputs
-        models = [0] * n_outputs
+        self.models = [0] * n_outputs
         rmse_train_spectral = np.zeros(n_outputs)
         for i, v in enumerate(self.domain.output_variables):
             # Training
-            models[i] = ThompsonSampledModel(v.name)
-            train_results[i] = models[i].fit(
+            self.models[i] = ThompsonSampledModel(v.name)
+            train_results[i] = self.models[i].fit(
                 inputs,
                 outputs[[v.name]],
                 n_retries=self.n_retries,
                 n_spectral_points=self.n_spectral_points,
             )
 
             # Evaluate spectral sampled functions
-            sample_f = lambda x: np.atleast_2d(models[i].rff(x)).T
+            sample_f = lambda x: np.atleast_2d(self.models[i].rff(x)).T
             rmse_train_spectral[i] = rmse(
                 sample_f(inputs.to_numpy().astype("float")),
                 outputs[[v.name]].to_numpy().astype("float"),
@@ -233,13 +231,13 @@ def suggest_experiments(self, num_experiments, prev_res: DataSet = None, **kwarg
         if (self.domain.num_continuous_dimensions() == 0) and (
             self.domain.num_categorical_variables() == 1
         ):
-            X, yhat = self._categorical_enumerate(models)
+            X, yhat = self._categorical_enumerate(self.models)
         # Mixed domains
         elif self.categorical_combos is not None and len(self.input_columns) > 1:
-            X, yhat = self._nsga_optimize_mixed(models)
+            X, yhat = self._nsga_optimize_mixed(self.models)
         # Continous domains
         elif self.categorical_combos is None and len(self.input_columns) > 0:
-            X, yhat = self._nsga_optimize(models)
+            X, yhat = self._nsga_optimize(self.models)
 
         # Return if no suggestiosn found
         if X.shape[0] == 0 and yhat.shape[0] == 0:
@@ -329,6 +327,9 @@ def _nsga_optimize_mixed(self, models):
             y = np.atleast_2d(self.internal_res.F).tolist()
             X = DataSet(X, columns=problem.X_columns)
             y = DataSet(y, columns=[v.name for v in self.domain.output_variables])
+            for v in self.domain.output_variables:
+                if v.maximize:
+                    y[v.name] = -y[v.name]
             # Add in categorical variables
             for key, value in combo.to_dict().items():
                 X[key] = value
@@ -581,13 +582,15 @@ def predict(self, X: DataSet, **kwargs):
         return self.rff(X)
 
     def save(self, filepath=None):
+        import pyrff
         if filepath is None:
             filepath = get_summit_config_path() / "tsemo" / str(self.uuid_val)
             os.makedirs(filepath, exist_ok=True)
             filepath = filepath / "models.h5"
         pyrff.save_rffs([self.rff], filepath)
 
     def load(self, filepath=None):
+        import pyrff
         if filepath is None:
             filepath = get_summit_config_path() / "tsemo" / str(self.uuid_val)
             os.makedirs(filepath, exist_ok=True)

summit/tests/test_runner.py

@@ -140,7 +140,7 @@ def test_runner_mo_integration(strategy, experiment):
         SOBO,
         TSEMO,
     ]:
-        # only run on strategies that work with categorical variables deireclty
+        # only run on strategies that work with categorical variables direclty
         return
     elif strategy == TSEMO:
         s = strategy(exp.domain)
@@ -154,7 +154,7 @@ def test_runner_mo_integration(strategy, experiment):
         s = strategy(exp.domain, transform=transform)
         iterations = 3
 
-    r = Runner(strategy=s, experiment=exp, max_iterations=iterations, batch_size=1)
+    r = Runner(strategy=s, experiment=exp, num_initial_experiments=8, max_iterations=iterations, batch_size=1)
     r.run()
 
     # Try saving and loading

summit/tests/test_strategies.py

@@ -6,6 +6,7 @@
 from fastprogress.fastprogress import progress_bar
 import numpy as np
 import os
+import matplotlib.pyplot as plt
 
 def test_strategy():
     class MockStrategy(Strategy):
@@ -692,10 +693,10 @@ def test_mtbo(
 
 
 @pytest.mark.parametrize("batch_size", [1, 2, 10])
-def test_tsemo(batch_size, test_num_improve_iter=2, save=False):
+@pytest.mark.parametrize("maximize", [True, False])
+def test_tsemo(batch_size, maximize, test_num_improve_iter=5, save=False, plot=True):
     num_inputs = 2
-    num_objectives = 2
-    lab = VLMOP2()
+    lab = VLMOP2(maximize=maximize)
     strategy = TSEMO(lab.domain)
     experiments = strategy.suggest_experiments(5 * num_inputs)
 
@@ -728,7 +729,7 @@ def test_tsemo(batch_size, test_num_improve_iter=2, save=False):
                 )
             )
             break
-    # assert hv > 117.0
+    assert hv > 110.0
 
 
 @pytest.mark.parametrize(