Add DiscretizingExperimenters for providing some Discrete/Categorical…

… parameters and allows for mixed ProblemStatements.

PiperOrigin-RevId: 488419426

vizier/_src/benchmarks/experimenters/discretizing_experimenter.py

@@ -0,0 +1,115 @@
+# Copyright 2022 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Experimenter that discretizes the parameters of search space."""
+
+import copy
+from typing import Sequence, Mapping
+
+from vizier import pyvizier
+from vizier._src.benchmarks.experimenters import experimenter
+
+
+class DiscretizingExperimenter(experimenter.Experimenter):
+  """DiscretizingExperimenter discretizes the parameters of search space."""
+
+  def __init__(self,
+               exptr: experimenter.Experimenter,
+               discretization: Mapping[str, pyvizier.MonotypeParameterSequence],
+               *,
+               allow_oov: bool = False):
+    """DiscretizingExperimenter discretizes continuous parameters.
+
+    Currently only supports flat double search spaces. Note that the discretized
+    parameters must fit within the bounds of the continuous parameters. This
+    also supports CATEGORICAL parameters but feasible categories must be
+    convertible to floats.
+
+    Args:
+      exptr: Underlying experimenter to be wrapped.
+      discretization: Dict of parameter name to discrete/categorical values.
+      allow_oov: Allows out of vocabulary values for Trial parameter in
+        Evaluate. If True, evaluate the underlying experimenter at any given
+        parameter values, whether feasible or not.
+
+    Raises:
+      ValueError: Non-double underlying parameters or discrete values OOB.
+    """
+    self._exptr = exptr
+    self._discretization = discretization
+    self._allow_oov = allow_oov
+    exptr_problem_statement = exptr.problem_statement()
+
+    if exptr_problem_statement.search_space.is_conditional:
+      raise ValueError('Search space should not have conditional'
+                       f' parameters  {exptr_problem_statement}')
+
+    search_params = exptr_problem_statement.search_space.parameters
+    param_names = [param.name for param in search_params]
+    for name in discretization.keys():
+      if name not in param_names:
+        raise ValueError(f'Parameter {name} not in search space'
+                         f' parameters for discretization: {search_params}')
+    new_parameter_configs = []
+    for parameter in search_params:
+      if parameter.name not in discretization:
+        new_parameter_configs.append(parameter)
+        continue
+
+      if parameter.type != pyvizier.ParameterType.DOUBLE:
+        raise ValueError(
+            f'Non-double parameters cannot be discretized {parameter}')
+      # Discretize the parameters.
+      min_value, max_value = parameter.bounds
+      for value in discretization[parameter.name]:
+        float_value = float(value)
+        if float_value > max_value or float_value < min_value:
+          raise ValueError(f'Discretized values are not in bounds {parameter}')
+      new_parameter_configs.append(
+          pyvizier.ParameterConfig.factory(
+              name=parameter.name,
+              feasible_values=discretization[parameter.name],
+              scale_type=parameter.scale_type,
+              external_type=parameter.external_type))
+
+    self._problem_statement = copy.deepcopy(exptr_problem_statement)
+    self._problem_statement.search_space = pyvizier.SearchSpace._factory(
+        new_parameter_configs)
+
+  def problem_statement(self) -> pyvizier.ProblemStatement:
+    return self._problem_statement
+
+  def evaluate(self, suggestions: Sequence[pyvizier.Trial]) -> None:
+    """Evaluate the trials after conversion to double."""
+
+    old_parameters = []
+    for suggestion in suggestions:
+      old_parameters.append(suggestion.parameters)
+      new_parameter_dict = {}
+      for name, param in suggestion.parameters.items():
+        if name in self._discretization:
+          if self._allow_oov:
+            if param.value not in self._discretization[name]:
+              raise ValueError(
+                  f'Parameter {param} not in {self._discretization[name]}')
+          new_parameter_dict[name] = param.as_float
+        else:
+          new_parameter_dict[name] = param
+      suggestion.parameters = pyvizier.ParameterDict(new_parameter_dict)
+    self._exptr.evaluate(suggestions)
+    for old_param, suggestion in zip(old_parameters, suggestions):
+      suggestion.parameters = old_param
+
+  def __repr__(self):
+    return f'DiscretizingExperimenter({self._discretization}) on {self._exptr}'

vizier/_src/benchmarks/experimenters/discretizing_experimenter_test.py

@@ -0,0 +1,83 @@
+# Copyright 2022 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+from vizier import pyvizier
+from vizier._src.benchmarks.experimenters import discretizing_experimenter
+from vizier._src.benchmarks.experimenters import numpy_experimenter
+from vizier._src.benchmarks.experimenters.synthetic import bbob
+
+from absl.testing import absltest
+from absl.testing import parameterized
+
+
+class DiscretizingExperimenterTest(parameterized.TestCase):
+
+  @parameterized.named_parameters(
+      ('Sphere', bbob.Sphere), ('Rastrigin', bbob.Rastrigin),
+      ('BuecheRastrigin', bbob.BuecheRastrigin),
+      ('LinearSlope', bbob.LinearSlope),
+      ('AttractiveSector', bbob.AttractiveSector),
+      ('StepEllipsoidal', bbob.StepEllipsoidal),
+      ('RosenbrockRotated', bbob.RosenbrockRotated), ('Discus', bbob.Discus),
+      ('BentCigar', bbob.BentCigar), ('SharpRidge', bbob.SharpRidge),
+      ('DifferentPowers', bbob.DifferentPowers),
+      ('Weierstrass', bbob.Weierstrass), ('SchaffersF7', bbob.SchaffersF7),
+      ('SchaffersF7IllConditioned', bbob.SchaffersF7IllConditioned),
+      ('GriewankRosenbrock', bbob.GriewankRosenbrock),
+      ('Schwefel', bbob.Schwefel), ('Katsuura', bbob.Katsuura),
+      ('Lunacek', bbob.Lunacek), ('Gallagher101Me', bbob.Gallagher101Me))
+  def testNumpyExperimenter(self, func):
+    dim = 3
+    exptr = numpy_experimenter.NumpyExperimenter(
+        func, bbob.DefaultBBOBProblemStatement(dim))
+
+    # Asserts parameters are the same.
+    parameters = list(exptr.problem_statement().search_space.parameters)
+    self.assertLen(parameters, dim)
+
+    discretization = {
+        parameters[0].name: ['-1', '0', '1'],
+        parameters[1].name: [0, 1, 2]
+    }
+
+    dis_exptr = discretizing_experimenter.DiscretizingExperimenter(
+        exptr, discretization)
+    discretized_parameters = dis_exptr.problem_statement(
+    ).search_space.parameters
+
+    self.assertLen(discretized_parameters, dim)
+    self.assertListEqual([p.type for p in discretized_parameters], [
+        pyvizier.ParameterType.CATEGORICAL, pyvizier.ParameterType.DISCRETE,
+        pyvizier.ParameterType.DOUBLE
+    ])
+
+    parameters = {
+        parameters[0].name: '0',
+        parameters[1].name: 1,
+        parameters[2].name: 1.5
+    }
+    t = pyvizier.Trial(parameters=parameters)
+
+    dis_exptr.evaluate([t])
+    metric_name = exptr.problem_statement().metric_information.item().name
+    self.assertAlmostEqual(
+        func(np.array([0.0, 1.0, 1.5])),
+        t.final_measurement.metrics[metric_name].value)
+    self.assertEqual(t.status, pyvizier.TrialStatus.COMPLETED)
+    self.assertDictEqual(t.parameters.as_dict(), parameters)
+
+
+if __name__ == '__main__':
+  absltest.main()