From 7728e98e8c0564cdb3a9bab6e54764a02bca6be9 Mon Sep 17 00:00:00 2001 From: Richard Zhang Date: Mon, 14 Nov 2022 11:17:05 -0800 Subject: [PATCH] Add DiscretizingExperimenters for providing some Discrete/Categorical parameters and allows for mixed ProblemStatements. PiperOrigin-RevId: 488419426 --- .../discretizing_experimenter.py | 115 ++++++++++++++++++ .../discretizing_experimenter_test.py | 83 +++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 vizier/_src/benchmarks/experimenters/discretizing_experimenter.py create mode 100644 vizier/_src/benchmarks/experimenters/discretizing_experimenter_test.py diff --git a/vizier/_src/benchmarks/experimenters/discretizing_experimenter.py b/vizier/_src/benchmarks/experimenters/discretizing_experimenter.py new file mode 100644 index 000000000..9e634199e --- /dev/null +++ b/vizier/_src/benchmarks/experimenters/discretizing_experimenter.py @@ -0,0 +1,115 @@ +# Copyright 2022 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Experimenter that discretizes the parameters of search space.""" + +import copy +from typing import Sequence, Mapping + +from vizier import pyvizier +from vizier._src.benchmarks.experimenters import experimenter + + +class DiscretizingExperimenter(experimenter.Experimenter): + """DiscretizingExperimenter discretizes the parameters of search space.""" + + def __init__(self, + exptr: experimenter.Experimenter, + discretization: Mapping[str, pyvizier.MonotypeParameterSequence], + *, + allow_oov: bool = False): + """DiscretizingExperimenter discretizes continuous parameters. + + Currently only supports flat double search spaces. Note that the discretized + parameters must fit within the bounds of the continuous parameters. This + also supports CATEGORICAL parameters but feasible categories must be + convertible to floats. + + Args: + exptr: Underlying experimenter to be wrapped. + discretization: Dict of parameter name to discrete/categorical values. + allow_oov: Allows out of vocabulary values for Trial parameter in + Evaluate. If True, evaluate the underlying experimenter at any given + parameter values, whether feasible or not. + + Raises: + ValueError: Non-double underlying parameters or discrete values OOB. + """ + self._exptr = exptr + self._discretization = discretization + self._allow_oov = allow_oov + exptr_problem_statement = exptr.problem_statement() + + if exptr_problem_statement.search_space.is_conditional: + raise ValueError('Search space should not have conditional' + f' parameters {exptr_problem_statement}') + + search_params = exptr_problem_statement.search_space.parameters + param_names = [param.name for param in search_params] + for name in discretization.keys(): + if name not in param_names: + raise ValueError(f'Parameter {name} not in search space' + f' parameters for discretization: {search_params}') + new_parameter_configs = [] + for parameter in search_params: + if parameter.name not in discretization: + new_parameter_configs.append(parameter) + continue + + if parameter.type != pyvizier.ParameterType.DOUBLE: + raise ValueError( + f'Non-double parameters cannot be discretized {parameter}') + # Discretize the parameters. + min_value, max_value = parameter.bounds + for value in discretization[parameter.name]: + float_value = float(value) + if float_value > max_value or float_value < min_value: + raise ValueError(f'Discretized values are not in bounds {parameter}') + new_parameter_configs.append( + pyvizier.ParameterConfig.factory( + name=parameter.name, + feasible_values=discretization[parameter.name], + scale_type=parameter.scale_type, + external_type=parameter.external_type)) + + self._problem_statement = copy.deepcopy(exptr_problem_statement) + self._problem_statement.search_space = pyvizier.SearchSpace._factory( + new_parameter_configs) + + def problem_statement(self) -> pyvizier.ProblemStatement: + return self._problem_statement + + def evaluate(self, suggestions: Sequence[pyvizier.Trial]) -> None: + """Evaluate the trials after conversion to double.""" + + old_parameters = [] + for suggestion in suggestions: + old_parameters.append(suggestion.parameters) + new_parameter_dict = {} + for name, param in suggestion.parameters.items(): + if name in self._discretization: + if self._allow_oov: + if param.value not in self._discretization[name]: + raise ValueError( + f'Parameter {param} not in {self._discretization[name]}') + new_parameter_dict[name] = param.as_float + else: + new_parameter_dict[name] = param + suggestion.parameters = pyvizier.ParameterDict(new_parameter_dict) + self._exptr.evaluate(suggestions) + for old_param, suggestion in zip(old_parameters, suggestions): + suggestion.parameters = old_param + + def __repr__(self): + return f'DiscretizingExperimenter({self._discretization}) on {self._exptr}' diff --git a/vizier/_src/benchmarks/experimenters/discretizing_experimenter_test.py b/vizier/_src/benchmarks/experimenters/discretizing_experimenter_test.py new file mode 100644 index 000000000..654ad26a0 --- /dev/null +++ b/vizier/_src/benchmarks/experimenters/discretizing_experimenter_test.py @@ -0,0 +1,83 @@ +# Copyright 2022 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from vizier import pyvizier +from vizier._src.benchmarks.experimenters import discretizing_experimenter +from vizier._src.benchmarks.experimenters import numpy_experimenter +from vizier._src.benchmarks.experimenters.synthetic import bbob + +from absl.testing import absltest +from absl.testing import parameterized + + +class DiscretizingExperimenterTest(parameterized.TestCase): + + @parameterized.named_parameters( + ('Sphere', bbob.Sphere), ('Rastrigin', bbob.Rastrigin), + ('BuecheRastrigin', bbob.BuecheRastrigin), + ('LinearSlope', bbob.LinearSlope), + ('AttractiveSector', bbob.AttractiveSector), + ('StepEllipsoidal', bbob.StepEllipsoidal), + ('RosenbrockRotated', bbob.RosenbrockRotated), ('Discus', bbob.Discus), + ('BentCigar', bbob.BentCigar), ('SharpRidge', bbob.SharpRidge), + ('DifferentPowers', bbob.DifferentPowers), + ('Weierstrass', bbob.Weierstrass), ('SchaffersF7', bbob.SchaffersF7), + ('SchaffersF7IllConditioned', bbob.SchaffersF7IllConditioned), + ('GriewankRosenbrock', bbob.GriewankRosenbrock), + ('Schwefel', bbob.Schwefel), ('Katsuura', bbob.Katsuura), + ('Lunacek', bbob.Lunacek), ('Gallagher101Me', bbob.Gallagher101Me)) + def testNumpyExperimenter(self, func): + dim = 3 + exptr = numpy_experimenter.NumpyExperimenter( + func, bbob.DefaultBBOBProblemStatement(dim)) + + # Asserts parameters are the same. + parameters = list(exptr.problem_statement().search_space.parameters) + self.assertLen(parameters, dim) + + discretization = { + parameters[0].name: ['-1', '0', '1'], + parameters[1].name: [0, 1, 2] + } + + dis_exptr = discretizing_experimenter.DiscretizingExperimenter( + exptr, discretization) + discretized_parameters = dis_exptr.problem_statement( + ).search_space.parameters + + self.assertLen(discretized_parameters, dim) + self.assertListEqual([p.type for p in discretized_parameters], [ + pyvizier.ParameterType.CATEGORICAL, pyvizier.ParameterType.DISCRETE, + pyvizier.ParameterType.DOUBLE + ]) + + parameters = { + parameters[0].name: '0', + parameters[1].name: 1, + parameters[2].name: 1.5 + } + t = pyvizier.Trial(parameters=parameters) + + dis_exptr.evaluate([t]) + metric_name = exptr.problem_statement().metric_information.item().name + self.assertAlmostEqual( + func(np.array([0.0, 1.0, 1.5])), + t.final_measurement.metrics[metric_name].value) + self.assertEqual(t.status, pyvizier.TrialStatus.COMPLETED) + self.assertDictEqual(t.parameters.as_dict(), parameters) + + +if __name__ == '__main__': + absltest.main()