add hpobench wrapper

docs/src/code/benchmark/task.rst

@@ -16,3 +16,4 @@ Task modules
    task/rosenbrock
    task/forrester
    task/profet
+   task/hpobench

docs/src/code/benchmark/task/hpobench.rst

@@ -0,0 +1,5 @@
+HPOBench
+=============================
+
+.. automodule:: orion.benchmark.task.hpobench
+   :members:

docs/src/user/benchmark.rst

@@ -27,6 +27,15 @@ Beside out of box :doc:`/code/benchmark/task` and :doc:`/code/benchmark/assessme
 you can also extend benchmark to add new ``Tasks`` with :doc:`/code/benchmark/task/base` and
 ``Assessments`` with :doc:`/code/benchmark/assessment/base`,
 
+To run benchmark with task :doc:`/code/benchmark/task/hpobench`, use ``pip install orion[hpobench]``
+to install the extra requirements. HPOBench provides local and containerized benchmarks, but different
+local benchmark could ask for total difference extra requirements. With ``pip install orion[hpobench]``,
+you will be able to run local benchmark ``benchmarks.ml.tabular_benchmark``.
+If you want to run other benchmarks in local, refer HPOBench `Run a Benchmark Locally`_. To run
+containerized benchmarks, you will need to install `singularity`_.
+
 Learn how to get start using benchmark in Orion with this `sample notebook`_.
 
+.. _Run a Benchmark Locally: https://github.com/automl/HPOBench#run-a-benchmark-locally
+.. _singularity: https://singularity.hpcng.org/admin-docs/master/installation.html
 .. _sample notebook: https://github.com/Epistimio/orion/tree/develop/examples/benchmark/benchmark_get_start.ipynb

setup.py

@@ -63,6 +63,10 @@
         "pymoo==0.5.0",
         "hebo @ git+https://github.com/huawei-noah/[email protected]#egg=hebo&subdirectory=HEBO",
     ],
+    "hpobench": [
+        "openml",
+        "hpobench @ git+https://github.com/automl/HPOBench.git@master#egg=hpobench",
+    ],
 }
 extras_require["all"] = sorted(set(sum(extras_require.values(), [])))
 

src/orion/algo/space/configspace.py

@@ -23,6 +23,7 @@
         IntegerHyperparameter,
         NormalFloatHyperparameter,
         NormalIntegerHyperparameter,
+        OrdinalHyperparameter,
         UniformFloatHyperparameter,
         UniformIntegerHyperparameter,
     )
@@ -44,6 +45,7 @@ class DummyType:
     UniformIntegerHyperparameter = DummyType
     NormalIntegerHyperparameter = DummyType
     CategoricalHyperparameter = DummyType
+    OrdinalHyperparameter = DummyType
 
 
 class UnsupportedPrior(Exception):
@@ -185,6 +187,13 @@ def _from_categorical(dim: CategoricalHyperparameter) -> Categorical:
     return Categorical(dim.name, choices)
 
 
+@to_oriondim.register(OrdinalHyperparameter)
+def _from_ordinal(dim: OrdinalHyperparameter) -> Categorical:
+    """Builds a categorical dimension from a categorical hyperparameter"""
+    choices = list(dim.sequence)
+    return Categorical(dim.name, choices)
+
+
 @to_oriondim.register(UniformIntegerHyperparameter)
 @to_oriondim.register(UniformFloatHyperparameter)
 def _from_uniform(dim: Hyperparameter) -> Integer | Real:

src/orion/benchmark/task/__init__.py

@@ -8,6 +8,7 @@
 from .carromtable import CarromTable
 from .eggholder import EggHolder
 from .forrester import Forrester
+from .hpobench import HPOBench
 from .rosenbrock import RosenBrock
 
 try:
@@ -25,6 +26,7 @@
     "EggHolder",
     "Forrester",
     "profet",
+    "HPOBench",
     # "ProfetSvmTask",
     # "ProfetFcNetTask",
     # "ProfetForresterTask",

src/orion/benchmark/task/hpobench.py

@@ -0,0 +1,86 @@
+"""
+Task for HPOBench
+=================
+"""
+import importlib
+import subprocess
+from typing import Dict, List, Union
+
+from orion.algo.space.configspace import to_orionspace
+from orion.benchmark.task.base import BenchmarkTask
+
+
+class HPOBench(BenchmarkTask):
+    """Benchmark Task wrapper over HPOBench (https://github.com/automl/HPOBench)
+
+    For more information on HPOBench, see original paper at https://arxiv.org/abs/2109.06716.
+
+    Katharina Eggensperger, Philipp Müller, Neeratyoy Mallik, Matthias Feurer, René Sass, Aaron Klein,
+    Noor Awad, Marius Lindauer, Frank Hutter. "HPOBench: A Collection of Reproducible Multi-Fidelity
+    Benchmark Problems for HPO" Thirty-fifth Conference on Neural Information Processing Systems
+    Datasets and Benchmarks Track (Round 2).
+
+    Parameters
+    ----------
+    max_trials : int
+        Maximum number of trials for this task.
+    hpo_benchmark_class : str
+        Full path to a particular class of benchmark in HPOBench.
+    benchmark_kwargs: str
+        Optional parameters to create benchmark instance of class `hpo_benchmark_class`.
+    objective_function_kwargs: dict
+        Optional parameters to use when calling `objective_function` of the benchmark instance.
+    """
+
+    def __init__(
+        self,
+        max_trials: int,
+        hpo_benchmark_class: Union[str, None] = None,
+        benchmark_kwargs: dict = dict(),
+        objective_function_kwargs: dict = dict(),
+    ):
+        super().__init__(
+            max_trials=max_trials,
+            hpo_benchmark_class=hpo_benchmark_class,
+            benchmark_kwargs=benchmark_kwargs,
+            objective_function_kwargs=objective_function_kwargs,
+        )
+        self._verify_benchmark(hpo_benchmark_class)
+        self.hpo_benchmark_cls = self._load_benchmark(hpo_benchmark_class)
+        self.benchmark_kwargs = benchmark_kwargs
+        self.objective_function_kwargs = objective_function_kwargs
+
+    def call(self, **kwargs) -> List[Dict]:
+        hpo_benchmark = self.hpo_benchmark_cls(**self.benchmark_kwargs)
+        result_dict = hpo_benchmark.objective_function(
+            configuration=kwargs, **self.objective_function_kwargs
+        )
+        objective = result_dict["function_value"]
+        return [
+            dict(
+                name=self.hpo_benchmark_cls.__name__, type="objective", value=objective
+            )
+        ]
+
+    def _load_benchmark(self, hpo_benchmark_class: str):
+        package, cls = hpo_benchmark_class.rsplit(".", 1)
+        module = importlib.import_module(package)
+        return getattr(module, cls)
+
+    def _verify_benchmark(self, hpo_benchmark_class: str):
+        if not hpo_benchmark_class:
+            raise AttributeError("Please provide full path to a HPOBench benchmark")
+        if "container" in hpo_benchmark_class:
+            code, message = subprocess.getstatusoutput("singularity -h")
+            if code != 0:
+                raise AttributeError(
+                    "Can not run conterized benchmark without Singularity: {}".format(
+                        message
+                    )
+                )
+
+    def get_search_space(self) -> Dict[str, str]:
+        configuration_space = self.hpo_benchmark_cls(
+            **self.benchmark_kwargs
+        ).get_configuration_space()
+        return to_orionspace(configuration_space)

tests/unittests/algo/test_configspace.py

@@ -52,10 +52,10 @@ def test_orion_configspace():
 
 def test_configspace_to_orion_unsupported():
     from ConfigSpace import ConfigurationSpace
-    from ConfigSpace.hyperparameters import OrdinalHyperparameter
+    from ConfigSpace.hyperparameters import Constant
 
     cspace = ConfigurationSpace()
-    cspace.add_hyperparameters([OrdinalHyperparameter("a", (1, 2, 0, 3))])
+    cspace.add_hyperparameters([Constant("a", 100)])
 
     with pytest.raises(NotImplementedError):
         _ = to_orionspace(cspace)

tests/unittests/benchmark/task/test_tasks.py

@@ -1,7 +1,19 @@
 #!/usr/bin/env python
 """Tests for :mod:`orion.benchmark.task`."""
+import inspect
 
-from orion.benchmark.task import Branin, CarromTable, EggHolder, RosenBrock
+import pytest
+
+from orion.algo.space import Space
+from orion.benchmark.task import Branin, CarromTable, EggHolder, HPOBench, RosenBrock
+
+try:
+    from hpobench import __version__
+
+    print(__version__)
+    IMPORT_ERROR = None
+except ImportError as err:
+    IMPORT_ERROR = err
 
 
 class TestBranin:
@@ -101,3 +113,72 @@ def test_search_space(self):
         """Test to get task search space"""
         task = RosenBrock(2)
         assert task.get_search_space() == {"x": "uniform(-5, 10, shape=2)"}
+
+
+@pytest.mark.skipif(
+    IMPORT_ERROR is not None,
+    reason="Running without HPOBench",
+)
+class TestHPOBench:
+    """Test benchmark task HPOBenchWrapper"""
+
+    def test_create_with_non_container_benchmark(self):
+        """Test to create HPOBench local benchmark"""
+        task = HPOBench(
+            max_trials=2,
+            hpo_benchmark_class="hpobench.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+            benchmark_kwargs=dict(model="xgb", task_id=168912),
+        )
+        assert task.max_trials == 2
+        assert inspect.isclass(task.hpo_benchmark_cls)
+        assert task.configuration == {
+            "HPOBench": {
+                "hpo_benchmark_class": "hpobench.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+                "benchmark_kwargs": {"model": "xgb", "task_id": 168912},
+                "objective_function_kwargs": {},
+                "max_trials": 2,
+            }
+        }
+
+    def test_create_with_container_benchmark(self):
+        """Test to create HPOBench container benchmark"""
+        with pytest.raises(AttributeError) as ex:
+            HPOBench(
+                max_trials=2,
+                hpo_benchmark_class="hpobench.container.benchmark.ml.tabular_benchmarks.TabularBenchmark",
+            )
+
+    def test_call(self):
+        """Test to run a local HPOBench benchmark"""
+        task = HPOBench(
+            max_trials=2,
+            hpo_benchmark_class="hpobench.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+            benchmark_kwargs=dict(model="xgb", task_id=168912),
+        )
+        params = {
+            "colsample_bytree": 1.0,
+            "eta": 0.045929204672575,
+            "max_depth": 1.0,
+            "reg_lambda": 10.079368591308594,
+        }
+
+        objectives = task(**params)
+        assert objectives == [
+            {
+                "name": "TabularBenchmark",
+                "type": "objective",
+                "value": 0.056373193166885674,
+            }
+        ]
+
+    def test_search_space(self):
+        """Test to get task search space"""
+        task = HPOBench(
+            max_trials=2,
+            hpo_benchmark_class="hpobench.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+            benchmark_kwargs=dict(model="xgb", task_id=168912),
+        )
+        space = task.get_search_space()
+
+        assert isinstance(space, Space)
+        assert len(space) == 4