From 4b2df6a9e11524588dfab384630fbadad0adfa80 Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Mon, 12 Jul 2021 16:01:55 -0500 Subject: [PATCH 01/11] Tentative CCS support. --- skopt/optimizer/optimizer.py | 24 ++++++++++--- skopt/space/space.py | 69 +++++++++++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/skopt/optimizer/optimizer.py b/skopt/optimizer/optimizer.py index c0430edc..eed24f00 100644 --- a/skopt/optimizer/optimizer.py +++ b/skopt/optimizer/optimizer.py @@ -4,6 +4,7 @@ from numbers import Number import ConfigSpace as CS +import cconfigspace as CCS import numpy as np import pandas as pd @@ -316,6 +317,11 @@ def __init__( if isinstance(self.base_estimator_, GaussianProcessRegressor): raise RuntimeError("GP estimator is not available with ConfigSpace!") + elif type(dimensions) is CCS.ConfigurationSpace: + self.ccs = dimensions + + if isinstance(self.base_estimator_, GaussianProcessRegressor): + raise RuntimeError("GP estimator is not available with CCS!") else: # normalize space if GP regressor @@ -386,10 +392,16 @@ def copy(self, random_state=None): Set the random state of the copy. """ + dimens = None + if hasattr(self, "config_space"): + dimens = self.config_space + elif hasattr(self, "ccs"): + dimens = self.ccs + else: + dimens = self.space.dimensions + optimizer = Optimizer( - dimensions=self.config_space - if hasattr(self, "config_space") - else self.space.dimensions, + dimensions=dimens, base_estimator=self.base_estimator_, n_initial_points=self.n_initial_points_, initial_point_generator=self._initial_point_generator, @@ -710,7 +722,7 @@ def _ask(self): next_x = self._next_x if next_x is not None: - if not self.space.is_config_space: + if not self.space.is_config_space and not self.space.is_ccs: min_delta_x = min( [self.space.distance(next_x, xi) for xi in self.Xi] ) @@ -751,6 +763,8 @@ def tell(self, x, y, fit=True): """ if self.space.is_config_space: pass + elif self.space.is_ccs: + pass else: check_x_in_space(x, self.space) @@ -924,7 +938,7 @@ def _tell(self, x, y, fit=True): # lbfgs should handle this but just in case there are # precision errors. if not self.space.is_categorical: - if not self.space.is_config_space: + if not self.space.is_config_space and not self.space.is_ccs: next_x = np.clip( next_x, transformed_bounds[:, 0], transformed_bounds[:, 1] ) diff --git a/skopt/space/space.py b/skopt/space/space.py index 406fe73a..63007945 100644 --- a/skopt/space/space.py +++ b/skopt/space/space.py @@ -30,6 +30,7 @@ import ConfigSpace as CS from ConfigSpace.util import deactivate_inactive_hyperparameters +import cconfigspace as CCS from sklearn.impute import SimpleImputer @@ -896,8 +897,11 @@ def __init__(self, dimensions, model_sdv=None): # attributes used when a ConfigurationSpace from ConfigSpace is given self.is_config_space = False + self.is_ccs = False self.config_space_samples = None + self.ccs_samples = None self.config_space_explored = False + self.ccs_explored = False self.imp_const = SimpleImputer( missing_values=np.nan, strategy="constant", fill_value=-1000 @@ -970,6 +974,52 @@ def __init__(self, dimensions, model_sdv=None): else: raise ValueError("Unknown Hyperparameter type.") dimensions = space + elif isinstance(dimensions, CCS.ConfigurationSpace): + self.is_ccs = True + self.ccs = dimensions + self.hps_type = {} + + hps = self.ccs.hyperparameters + cond_hps = [x.name for x in self.ccs.conditional_hyperparameters] + + space = [] + for x in hps: + self.hps_names.append(x.name) + distrib = self.ccs.get_hyperparameter_distribution(x)[0] + if (isinstance(x, CCS.CategoricalHyperparameter) or + isinstance(x, CCS.OrdinalHyperparameter) or + isinstance(x, CCS.DiscreteHyperparameter)): + vals = list(x.values) + if x.name in cond_hps: + vals.append("NA") + if isinstance(distrib, CCS.RouletteDistribution): + param = Categorical(vals, prior=distrib.areas, name=x.name) + elif sinstance(distrib, CCS.UniformDistribution): + param = Categorical(vals, name=x.name) + else: + raise ValueError("Unsupported distribution") + space.append(param) + self.hps_type[x.name] = "Categorical" + elif isinstance(x, CCS.NumericalHyperparameter): + prior = "uniform" + lower = x.lower + upper = x.upper + t = x.data_type + if isinstance(distrib, CCS.UniformDistribution): + if distrib.scale_type == CCS.ccs_scale_type.LOGARITHMIC: + prior = "log-uniform" + else: + raise ValueError("Unsupported distribution") + if CCS.ccs_numeric_type.NUM_INTEGER: + param = Integer(lower, upper, prior=prior, name=x.name) + self.hps_type[x.name] = "Integer" + else: + param = Real(lower, upper, prior=prior, name=x.name) + self.hps_type[x.name] = "Real" + space.append(param) + else: + raise ValueError("Unknown Hyperparameter type") + dimensions = space self.dimensions = [check_dimension(dim) for dim in dimensions] def __eq__(self, other): @@ -1149,6 +1199,23 @@ def rvs(self, n_samples=1, random_state=None): req_points.append(point) return req_points + elif self.is_ccs: + confs = self.ccs.samples(n_samples) + hps = self.ccs.hyperparameters + points = [] + for conf in confs: + point = [] + for hp in hps: + val = conf.value(hp) + if ccs.ccs_inactive == val: + if self.hps_type[hp.name] == "Categorical": + val = "NA" + else: + val = np.nan + point.append(val) + points.append(point) + + return points else: if self.model_sdv is None: # Draw @@ -1240,7 +1307,7 @@ def transform(self, X): # Repack as an array Xt = np.hstack([np.asarray(c).reshape((len(X), -1)) for c in columns]) - if False and self.is_config_space: + if False and (self.is_config_space or self.is_ccs): self.imp_const.fit(Xt) Xtt = self.imp_const.transform(Xt) Xt = Xtt From 0e206b88bb05e024c900243c1e2ef5ca354f4440 Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Fri, 20 Aug 2021 11:36:06 -0500 Subject: [PATCH 02/11] Updated to support normal prior. --- skopt/space/space.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/skopt/space/space.py b/skopt/space/space.py index 63007945..15008093 100644 --- a/skopt/space/space.py +++ b/skopt/space/space.py @@ -1008,6 +1008,10 @@ def __init__(self, dimensions, model_sdv=None): if isinstance(distrib, CCS.UniformDistribution): if distrib.scale_type == CCS.ccs_scale_type.LOGARITHMIC: prior = "log-uniform" + elif isinstance(distrib, CCS.NormalDistribution): + prior = "normal" + if distrib.scale_type == CCS.ccs_scale_type.LOGARITHMIC: + raise ValueError("Unsupported 'log' transformation for CCS.NumericalHyperparameter with normal prior.") else: raise ValueError("Unsupported distribution") if CCS.ccs_numeric_type.NUM_INTEGER: From b7a5ee8fe1d16a96f141f7b674353e50dab2ffcf Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Fri, 3 Sep 2021 15:21:17 -0500 Subject: [PATCH 03/11] Fix typos. --- skopt/space/space.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skopt/space/space.py b/skopt/space/space.py index 15008093..229136ee 100644 --- a/skopt/space/space.py +++ b/skopt/space/space.py @@ -994,7 +994,7 @@ def __init__(self, dimensions, model_sdv=None): vals.append("NA") if isinstance(distrib, CCS.RouletteDistribution): param = Categorical(vals, prior=distrib.areas, name=x.name) - elif sinstance(distrib, CCS.UniformDistribution): + elif isinstance(distrib, CCS.UniformDistribution): param = Categorical(vals, name=x.name) else: raise ValueError("Unsupported distribution") @@ -1211,7 +1211,7 @@ def rvs(self, n_samples=1, random_state=None): point = [] for hp in hps: val = conf.value(hp) - if ccs.ccs_inactive == val: + if CCS.ccs_inactive == val: if self.hps_type[hp.name] == "Categorical": val = "NA" else: From 6b3687a797fa053120f710d894041045e1371f4c Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Tue, 23 Nov 2021 10:44:57 -0600 Subject: [PATCH 04/11] Made CCS support optional. --- skopt/optimizer/optimizer.py | 10 ++++++++-- skopt/space/space.py | 9 +++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/skopt/optimizer/optimizer.py b/skopt/optimizer/optimizer.py index eed24f00..9530cd25 100644 --- a/skopt/optimizer/optimizer.py +++ b/skopt/optimizer/optimizer.py @@ -4,7 +4,13 @@ from numbers import Number import ConfigSpace as CS -import cconfigspace as CCS +ccs_active = False +try: + import cconfigspace as CCS + ccs_active = True +except ImportError as a: + warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning) + import numpy as np import pandas as pd @@ -317,7 +323,7 @@ def __init__( if isinstance(self.base_estimator_, GaussianProcessRegressor): raise RuntimeError("GP estimator is not available with ConfigSpace!") - elif type(dimensions) is CCS.ConfigurationSpace: + elif ccs_active and isinstance(dimensions, CCS.ConfigurationSpace): self.ccs = dimensions if isinstance(self.base_estimator_, GaussianProcessRegressor): diff --git a/skopt/space/space.py b/skopt/space/space.py index 229136ee..6542c4fa 100644 --- a/skopt/space/space.py +++ b/skopt/space/space.py @@ -30,7 +30,12 @@ import ConfigSpace as CS from ConfigSpace.util import deactivate_inactive_hyperparameters -import cconfigspace as CCS +ccs_active = False +try: + import cconfigspace as CCS + ccs_active = True +except ImportError as a: + warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning) from sklearn.impute import SimpleImputer @@ -974,7 +979,7 @@ def __init__(self, dimensions, model_sdv=None): else: raise ValueError("Unknown Hyperparameter type.") dimensions = space - elif isinstance(dimensions, CCS.ConfigurationSpace): + elif ccs_active && isinstance(dimensions, CCS.ConfigurationSpace): self.is_ccs = True self.ccs = dimensions self.hps_type = {} From 7526f05b1f2e53726b83ce59eb0c50987a6b148b Mon Sep 17 00:00:00 2001 From: Jaehoon Koo Date: Tue, 23 Nov 2021 15:02:35 -0600 Subject: [PATCH 05/11] fix CCS support optional --- skopt/space/space.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skopt/space/space.py b/skopt/space/space.py index 6542c4fa..590f2498 100644 --- a/skopt/space/space.py +++ b/skopt/space/space.py @@ -979,7 +979,7 @@ def __init__(self, dimensions, model_sdv=None): else: raise ValueError("Unknown Hyperparameter type.") dimensions = space - elif ccs_active && isinstance(dimensions, CCS.ConfigurationSpace): + elif ccs_active & isinstance(dimensions, CCS.ConfigurationSpace): self.is_ccs = True self.ccs = dimensions self.hps_type = {} From b5ecafc01f40c8a3f80e1ddd29ab16be5f174f3e Mon Sep 17 00:00:00 2001 From: Jaehoon Koo Date: Tue, 23 Nov 2021 15:16:22 -0600 Subject: [PATCH 06/11] fix CCS support optional --- skopt/optimizer/optimizer.py | 3 ++- skopt/space/space.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/skopt/optimizer/optimizer.py b/skopt/optimizer/optimizer.py index 9530cd25..239a4d65 100644 --- a/skopt/optimizer/optimizer.py +++ b/skopt/optimizer/optimizer.py @@ -9,7 +9,8 @@ import cconfigspace as CCS ccs_active = True except ImportError as a: - warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning) + import warnings + warnings.warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning) import numpy as np import pandas as pd diff --git a/skopt/space/space.py b/skopt/space/space.py index 590f2498..719125b9 100644 --- a/skopt/space/space.py +++ b/skopt/space/space.py @@ -35,7 +35,8 @@ import cconfigspace as CCS ccs_active = True except ImportError as a: - warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning) + import warnings + warnings.warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning) from sklearn.impute import SimpleImputer From 6f48a74b28aa5e492fa6d78d6816b26e087960fc Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Tue, 23 Nov 2021 15:55:40 -0600 Subject: [PATCH 07/11] Minor fixes and improvement to CCS support. --- skopt/optimizer/optimizer.py | 1 - skopt/space/space.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/skopt/optimizer/optimizer.py b/skopt/optimizer/optimizer.py index 239a4d65..6640621b 100644 --- a/skopt/optimizer/optimizer.py +++ b/skopt/optimizer/optimizer.py @@ -9,7 +9,6 @@ import cconfigspace as CCS ccs_active = True except ImportError as a: - import warnings warnings.warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning) import numpy as np diff --git a/skopt/space/space.py b/skopt/space/space.py index 719125b9..e84bab1f 100644 --- a/skopt/space/space.py +++ b/skopt/space/space.py @@ -980,7 +980,7 @@ def __init__(self, dimensions, model_sdv=None): else: raise ValueError("Unknown Hyperparameter type.") dimensions = space - elif ccs_active & isinstance(dimensions, CCS.ConfigurationSpace): + elif ccs_active and isinstance(dimensions, CCS.ConfigurationSpace): self.is_ccs = True self.ccs = dimensions self.hps_type = {} From ac4f414797882ac8c46761fa2874589c54a4a174 Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Tue, 23 Nov 2021 16:38:11 -0600 Subject: [PATCH 08/11] Check for the space type to get hyperparameters' names. --- skopt/optimizer/optimizer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skopt/optimizer/optimizer.py b/skopt/optimizer/optimizer.py index 6640621b..9b20f1d4 100644 --- a/skopt/optimizer/optimizer.py +++ b/skopt/optimizer/optimizer.py @@ -619,6 +619,8 @@ def _filter_duplicated(self, samples): if hasattr(self, "config_space"): hps_names = self.config_space.get_hyperparameter_names() + elif hasattr(self, "ccs"): + hps_names = [x.name for x in self.ccs.hyperparameters] else: hps_names = self.space.dimension_names From 0457838a233ab8bb1861e475bc7ff1cf24119332 Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Tue, 23 Nov 2021 16:48:14 -0600 Subject: [PATCH 09/11] Handle the CCS library not being found on the system. --- skopt/optimizer/optimizer.py | 2 +- skopt/space/space.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/skopt/optimizer/optimizer.py b/skopt/optimizer/optimizer.py index 9b20f1d4..21919b2e 100644 --- a/skopt/optimizer/optimizer.py +++ b/skopt/optimizer/optimizer.py @@ -8,7 +8,7 @@ try: import cconfigspace as CCS ccs_active = True -except ImportError as a: +except (ImportError, OSError) as a: warnings.warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning) import numpy as np diff --git a/skopt/space/space.py b/skopt/space/space.py index e84bab1f..68602c13 100644 --- a/skopt/space/space.py +++ b/skopt/space/space.py @@ -34,7 +34,7 @@ try: import cconfigspace as CCS ccs_active = True -except ImportError as a: +except (ImportError, OSError) as a: import warnings warnings.warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning) From 7590a77520c52530a2ab0278da50bee1c6703979 Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Thu, 2 Jun 2022 14:38:31 -0500 Subject: [PATCH 10/11] Optimize CCS code path. --- skopt/space/space.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/skopt/space/space.py b/skopt/space/space.py index 68602c13..694b308e 100644 --- a/skopt/space/space.py +++ b/skopt/space/space.py @@ -1215,8 +1215,9 @@ def rvs(self, n_samples=1, random_state=None): points = [] for conf in confs: point = [] - for hp in hps: - val = conf.value(hp) + values = conf.values + for i, hp in enumerate(hps): + val = values[i] if CCS.ccs_inactive == val: if self.hps_type[hp.name] == "Categorical": val = "NA" From aef606430d5e82c7daef34a7e74091d66a1aab37 Mon Sep 17 00:00:00 2001 From: Brice Videau Date: Thu, 2 Jun 2022 15:27:52 -0500 Subject: [PATCH 11/11] Set CCS rng state. --- skopt/optimizer/optimizer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/skopt/optimizer/optimizer.py b/skopt/optimizer/optimizer.py index 21919b2e..52c950d7 100644 --- a/skopt/optimizer/optimizer.py +++ b/skopt/optimizer/optimizer.py @@ -325,6 +325,7 @@ def __init__( raise RuntimeError("GP estimator is not available with ConfigSpace!") elif ccs_active and isinstance(dimensions, CCS.ConfigurationSpace): self.ccs = dimensions + self.ccs.rng.seed = self.rng.get_state()[1][0] if isinstance(self.base_estimator_, GaussianProcessRegressor): raise RuntimeError("GP estimator is not available with CCS!")