Merge pull request #113 from automl/configspace

Configspace definitions

dacbench/abstract_benchmark.py

@@ -1,6 +1,5 @@
 import json
 from types import FunctionType
-from typing import Callable
 
 import numpy as np
 from gym import spaces
@@ -10,13 +9,12 @@
 # from dacbench import ModuleLogger
 
 
-
 class AbstractBenchmark:
     """
     Abstract template for benchmark classes
     """
 
-    def __init__(self, config_path=None, config:  'objdict'=None):
+    def __init__(self, config_path=None, config: "objdict" = None):
         """
         Initialize benchmark class
 
@@ -71,6 +69,10 @@ def serialize_config(self):
         if "action_space" in self.config:
             conf["action_space"] = self.space_to_list(conf["action_space"])
 
+        # TODO: how can we use the built in serialization of configspace here?
+        if "config_space" in self.config:
+            conf["config_space"] = self.process_configspace(self.config.config_space)
+
         conf = AbstractBenchmark.__stringify_functions(conf)
 
         for k in self.config.keys():
@@ -97,11 +99,94 @@ def serialize_config(self):
 
         return conf
 
+    def process_configspace(self, configuration_space):
+        """
+        This is largely the builting cs.json.write method, but doesn't save the result directly
+        If this is ever implemented in cs, we can replace this method
+        """
+        from ConfigSpace.configuration_space import ConfigurationSpace
+        from ConfigSpace.hyperparameters import (
+            CategoricalHyperparameter,
+            UniformIntegerHyperparameter,
+            UniformFloatHyperparameter,
+            NormalIntegerHyperparameter,
+            NormalFloatHyperparameter,
+            OrdinalHyperparameter,
+            Constant,
+            UnParametrizedHyperparameter,
+        )
+        from ConfigSpace.read_and_write.json import (
+            _build_constant,
+            _build_condition,
+            _build_ordinal,
+            _build_forbidden,
+            _build_categorical,
+            _build_normal_int,
+            _build_normal_float,
+            _build_uniform_float,
+            _build_uniform_int,
+            _build_unparametrized_hyperparameter,
+        )
+
+        if not isinstance(configuration_space, ConfigurationSpace):
+            raise TypeError(
+                "pcs_parser.write expects an instance of %s, "
+                "you provided '%s'" % (ConfigurationSpace, type(configuration_space))
+            )
+
+        hyperparameters = []
+        conditions = []
+        forbiddens = []
+
+        for hyperparameter in configuration_space.get_hyperparameters():
+
+            if isinstance(hyperparameter, Constant):
+                hyperparameters.append(_build_constant(hyperparameter))
+            elif isinstance(hyperparameter, UnParametrizedHyperparameter):
+                hyperparameters.append(
+                    _build_unparametrized_hyperparameter(hyperparameter)
+                )
+            elif isinstance(hyperparameter, UniformFloatHyperparameter):
+                hyperparameters.append(_build_uniform_float(hyperparameter))
+            elif isinstance(hyperparameter, NormalFloatHyperparameter):
+                hyperparameters.append(_build_normal_float(hyperparameter))
+            elif isinstance(hyperparameter, UniformIntegerHyperparameter):
+                hyperparameters.append(_build_uniform_int(hyperparameter))
+            elif isinstance(hyperparameter, NormalIntegerHyperparameter):
+                hyperparameters.append(_build_normal_int(hyperparameter))
+            elif isinstance(hyperparameter, CategoricalHyperparameter):
+                hyperparameters.append(_build_categorical(hyperparameter))
+            elif isinstance(hyperparameter, OrdinalHyperparameter):
+                hyperparameters.append(_build_ordinal(hyperparameter))
+            else:
+                raise TypeError(
+                    "Unknown type: %s (%s)" % (type(hyperparameter), hyperparameter,)
+                )
+
+        for condition in configuration_space.get_conditions():
+            conditions.append(_build_condition(condition))
+
+        for forbidden_clause in configuration_space.get_forbiddens():
+            forbiddens.append(_build_forbidden(forbidden_clause))
+
+        rval = {}
+        if configuration_space.name is not None:
+            rval["name"] = configuration_space.name
+        rval["hyperparameters"] = hyperparameters
+        rval["conditions"] = conditions
+        rval["forbiddens"] = forbiddens
+
+        return rval
+
     @staticmethod
     def from_json(json_config):
         config = objdict(json.loads(json_config))
+        if "config_space" in config.keys():
+            from ConfigSpace.read_and_write import json as cs_json
+
+            config.config_space = cs_json.read(config.config_space)
         return AbstractBenchmark(config=config)
-    
+
     def to_json(self):
         conf = self.serialize_config()
         return json.dumps(conf)
@@ -155,7 +240,7 @@ def dejson_wrappers(self, wrapper_list):
             self.wrap_funcs.append(partial(func, *args))
 
     @staticmethod
-    def __import_from(module : str, name : str):
+    def __import_from(module: str, name: str):
         """
         Imports the class / function / ... with name from module
         Parameters
@@ -171,7 +256,7 @@ def __import_from(module : str, name : str):
         return getattr(module, name)
 
     @staticmethod
-    def __decorate_config_with_functions(conf : dict):
+    def __decorate_config_with_functions(conf: dict):
         """
         Replaced the stringified functions with the callable objects
         Parameters
@@ -182,13 +267,17 @@ def __decorate_config_with_functions(conf : dict):
         -------
 
         """
-        for key, value in {k: v for k, v in conf.items() if isinstance(v, list) and len(v) == 3 and v[0] == 'function'}.items():
+        for key, value in {
+            k: v
+            for k, v in conf.items()
+            if isinstance(v, list) and len(v) == 3 and v[0] == "function"
+        }.items():
             _, module_name, function_name = value
             conf[key] = AbstractBenchmark.__import_from(module_name, function_name)
         return conf
 
     @staticmethod
-    def __stringify_functions(conf : dict) -> dict:
+    def __stringify_functions(conf: dict) -> dict:
         """
         Replaced all callables in the config with a
         triple ('function', module_name, function_name)
@@ -201,12 +290,10 @@ def __stringify_functions(conf : dict) -> dict:
         -------
         modified dict
         """
-        for key, value in {k: v for k, v in conf.items() if isinstance(v, FunctionType)}.items():
-            conf[key] = [
-                'function',
-                conf[key].__module__,
-                conf[key].__name__
-            ]
+        for key, value in {
+            k: v for k, v in conf.items() if isinstance(v, FunctionType)
+        }.items():
+            conf[key] = ["function", conf[key].__module__, conf[key].__name__]
         return conf
 
     def space_to_list(self, space):
@@ -249,7 +336,9 @@ def jsonify_dict_space(self, dict_space):
             keys.append(k)
             value = dict_space[k]
             if not isinstance(value, (spaces.Box, spaces.Discrete)):
-                raise ValueError(f"Only Dict spaces made up of Box spaces or discrete spaces are supported but got {type(value)}")
+                raise ValueError(
+                    f"Only Dict spaces made up of Box spaces or discrete spaces are supported but got {type(value)}"
+                )
 
             if isinstance(value, spaces.Box):
                 low = value.low.tolist()
@@ -271,11 +360,13 @@ def dictify_json(self, dict_list):
             elif type == "discrete":
                 dict_space[type] = spaces.Discrete(*prepared_args)
             else:
-                raise TypeError(f"Currently only Discrete and Box spaces are allowed in Dict spaces got {type}")
+                raise TypeError(
+                    f"Currently only Discrete and Box spaces are allowed in Dict spaces got {type}"
+                )
 
         return dict_space
 
-    def load_config(self, config :  'objdict'):
+    def load_config(self, config: "objdict"):
         self.config = config
         if "observation_space_type" in self.config:
             # Types have to be numpy dtype (for gym spaces)s
@@ -325,9 +416,8 @@ def read_config_file(self, path):
         """
         with open(path, "r") as fp:
             config = objdict(json.load(fp))
-        
+
         self.load_config(config)
-
 
     def get_environment(self):
         """

dacbench/abstract_env.py

@@ -84,7 +84,42 @@ def __init__(self, config):
                     )
                     raise TypeError
 
-        if "action_space" in config.keys():
+        # TODO: use dicts by default for actions and observations
+        # The config could change this for RL purposes
+        if "config_space" in config.keys():
+            actions = config["config_space"].get_hyperparameters()
+            action_types = [type(a).__name__ for a in actions]
+
+            # Uniform action space
+            if all(t == action_types[0] for t in action_types):
+                if "Float" in action_types[0]:
+                    low = np.array([a.lower for a in actions])
+                    high = np.array([a.upper for a in actions])
+                    self.action_space = gym.spaces.Box(low=low, high=high)
+                elif "Integer" in action_types[0] or "Categorical" in action_types[0]:
+                    if len(action_types) == 1:
+                        try:
+                            n = actions[0].upper - actions[0].lower
+                        except:
+                            n = len(actions[0].choices)
+                        self.action_space = gym.spaces.Discrete(n)
+                    else:
+                        ns = []
+                        for a in actions:
+                            try:
+                                ns.append(a.upper - a.lower)
+                            except:
+                                ns.append(len(a.choices))
+                        self.action_space = gym.spaces.MultiDiscrete(np.array(ns))
+                else:
+                    raise ValueError(
+                        "Only float, integer and categorical hyperparameters are supported as of now"
+                    )
+            # Mixed action space
+            # TODO: implement this
+            else:
+                raise ValueError("Mixed type config spaces are currently not supported")
+        elif "action_space" in config.keys():
             self.action_space = config["action_space"]
         else:
             try:

dacbench/benchmarks/cma_benchmark.py

@@ -4,10 +4,16 @@
 import numpy as np
 import os
 import csv
+import ConfigSpace as CS
+import ConfigSpace.hyperparameters as CSH
 
 HISTORY_LENGTH = 40
 INPUT_DIM = 10
 
+DEFAULT_CFG_SPACE = CS.ConfigurationSpace()
+STEP_SIZE = CSH.UniformFloatHyperparameter(name='Step_size', lower=0, upper=10)
+DEFAULT_CFG_SPACE.add_hyperparameter(STEP_SIZE)
+
 INFO = {
     "identifier": "CMA-ES",
     "name": "Step-size adaption in CMA-ES",
@@ -26,6 +32,7 @@
     {
         "action_space_class": "Box",
         "action_space_args": [np.array([0]), np.array([10])],
+        "config_space": DEFAULT_CFG_SPACE,
         "observation_space_class": "Dict",
         "observation_space_type": None,
         "observation_space_args": [

dacbench/benchmarks/fast_downward_benchmark.py

@@ -3,12 +3,18 @@
 
 import numpy as np
 import os
+import ConfigSpace as CS
+import ConfigSpace.hyperparameters as CSH
 
 HEURISTICS = [
     "tiebreaking([pdb(pattern=manual_pattern([0,1])),weight(g(),-1)])",
     "tiebreaking([pdb(pattern=manual_pattern([0,2])),weight(g(),-1)])",
 ]
 
+DEFAULT_CFG_SPACE = CS.ConfigurationSpace()
+HEURISTIC = CSH.CategoricalHyperparameter(name='heuristic', choices=["toy1", "toy2"])
+DEFAULT_CFG_SPACE.add_hyperparameter(HEURISTIC)
+
 INFO = {
     "identifier": "FastDownward",
     "name": "Heuristic Selection for the FastDownward Planner",
@@ -30,8 +36,7 @@
 FD_DEFAULTS = objdict(
     {
         "heuristics": HEURISTICS,
-        "action_space_class": "Discrete",
-        "action_space_args": [len(HEURISTICS)],
+        "config_space": DEFAULT_CFG_SPACE, 
         "observation_space_class": "Box",
         "observation_space_type": np.float32,
         "observation_space_args": [

dacbench/benchmarks/luby_benchmark.py

@@ -5,11 +5,17 @@
 import numpy as np
 import os
 import csv
+import ConfigSpace as CS
+import ConfigSpace.hyperparameters as CSH
 
 MAX_STEPS = 2 ** 6
 LUBY_SEQUENCE = np.log2([next(luby_gen(i)) for i in range(1, 2 * MAX_STEPS + 2)])
 HISTORY_LENGTH = 5
 
+DEFAULT_CFG_SPACE = CS.ConfigurationSpace()
+SEQ = CSH.UniformIntegerHyperparameter(name='sequence_element', lower=0, upper=np.log2(MAX_STEPS))
+DEFAULT_CFG_SPACE.add_hyperparameter(SEQ)
+
 INFO = {
     "identifier": "Luby",
     "name": "Luby Sequence Approximation",
@@ -26,8 +32,7 @@
 
 LUBY_DEFAULTS = objdict(
     {
-        "action_space_class": "Discrete",
-        "action_space_args": [int(np.log2(MAX_STEPS))],
+        "config_space": DEFAULT_CFG_SPACE,
         "observation_space_class": "Box",
         "observation_space_type": np.float32,
         "observation_space_args": [

dacbench/benchmarks/modcma_benchmark.py

@@ -7,6 +7,33 @@
 from dacbench.abstract_benchmark import AbstractBenchmark, objdict
 from dacbench.envs import ModCMAEnv, CMAStepSizeEnv
 
+import ConfigSpace as CS
+import ConfigSpace.hyperparameters as CSH
+
+DEFAULT_CFG_SPACE = CS.ConfigurationSpace()
+ACTIVE = CSH.CategoricalHyperparameter(name='0_active', choices=[True, False])
+ELITIST = CSH.CategoricalHyperparameter(name='1_elitist', choices=[True, False])
+ORTHOGONAL = CSH.CategoricalHyperparameter(name='2_orthogonal', choices=[True, False])
+SEQUENTIAL = CSH.CategoricalHyperparameter(name='3_sequential', choices=[True, False])
+THRESHOLD_CONVERGENCE = CSH.CategoricalHyperparameter(name='4_threshold_convergence', choices=[True, False])
+STEP_SIZE_ADAPTION = CSH.CategoricalHyperparameter(name='5_step_size_adaption', choices=["csa", "tpa", "msr", "xnes", "m-xnes", "lp-xnes", "psr"])
+MIRRORED = CSH.CategoricalHyperparameter(name='6_mirrored', choices=["None", "mirrored", "mirrored pairwise"])
+BASE_SAMPLER = CSH.CategoricalHyperparameter(name='7_base_sampler', choices=["gaussian", "sobol", "halton"])
+WEIGHTS_OPTION = CSH.CategoricalHyperparameter(name='8_weights_option', choices=["default", "equal", "1/2^lambda"])
+LOCAL_RESTART = CSH.CategoricalHyperparameter(name='90_local_restart', choices=["None", "IPOP", "BIPOP"])
+BOUND_CORRECTION = CSH.CategoricalHyperparameter(name='91_bound_correction', choices=["None", "saturate", "unif_resample", "COTN", "toroidal", "mirror"])
+
+DEFAULT_CFG_SPACE.add_hyperparameter(ACTIVE)
+DEFAULT_CFG_SPACE.add_hyperparameter(ELITIST)
+DEFAULT_CFG_SPACE.add_hyperparameter(ORTHOGONAL)
+DEFAULT_CFG_SPACE.add_hyperparameter(SEQUENTIAL)
+DEFAULT_CFG_SPACE.add_hyperparameter(THRESHOLD_CONVERGENCE)
+DEFAULT_CFG_SPACE.add_hyperparameter(STEP_SIZE_ADAPTION)
+DEFAULT_CFG_SPACE.add_hyperparameter(MIRRORED)
+DEFAULT_CFG_SPACE.add_hyperparameter(BASE_SAMPLER)
+DEFAULT_CFG_SPACE.add_hyperparameter(WEIGHTS_OPTION)
+DEFAULT_CFG_SPACE.add_hyperparameter(LOCAL_RESTART)
+DEFAULT_CFG_SPACE.add_hyperparameter(BOUND_CORRECTION)
 
 INFO = {
     "identifier": "ModCMA",
@@ -24,6 +51,7 @@
 
 MODCMA_DEFAULTS = objdict(
     {
+        "config_space": DEFAULT_CFG_SPACE,
         "action_space_class": "MultiDiscrete",
         "action_space_args": [
             list(

dacbench/benchmarks/onell_benchmark.py

@@ -4,6 +4,7 @@
 import os
 import pandas as pd
 
+#TODO: adapt benchmark to fit the others (info etc.)
 
 class OneLLBenchmark(AbstractBenchmark):
     """