From 45daebb7ba619ec92c44690654b997017f81e2e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20B=C3=A1rtek?= Date: Thu, 23 Sep 2021 12:39:34 +0200 Subject: [PATCH] Handle invalid neighbors in local search gracefully (#773) * Catch ValueError raised by neighborhood iterator If the iterator returned by `ConfigSpace.util.get_one_exchange_neighborhood` reaches an invalid configuration, it raises a `ValueError` with the probability 5 %. The randomness is sampled using `np.random.random()`, so it is non-deterministic. See ConfigSpace/util.pyx:218. This commit ensures that the exception is caught and handled gracefully. * Consider only valid configurations in local search In local search, only advance to a better-scoring neighbor if that neighbor is a valid configuration. Note that `neighborhood_iterator` may yield an invalid configuration. --- smac/configspace/__init__.py | 4 +++- smac/optimizer/ei_optimization.py | 39 ++++++++++++++++++++----------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/smac/configspace/__init__.py b/smac/configspace/__init__.py index 40f32ded5..79a1fdf76 100644 --- a/smac/configspace/__init__.py +++ b/smac/configspace/__init__.py @@ -3,6 +3,7 @@ from ConfigSpace import ConfigurationSpace, Configuration, Constant, \ CategoricalHyperparameter, UniformFloatHyperparameter, \ UniformIntegerHyperparameter, InCondition +from ConfigSpace.exceptions import ForbiddenValueError from ConfigSpace.read_and_write import pcs, pcs_new, json from ConfigSpace.util import get_one_exchange_neighbourhood from smac.configspace.util import convert_configurations_to_array @@ -22,7 +23,8 @@ "pcs_new", "json", "get_one_exchange_neighbourhood", - "convert_configurations_to_array" + "convert_configurations_to_array", + "ForbiddenValueError" ] get_one_exchange_neighbourhood = partial(get_one_exchange_neighbourhood, stdev=0.05, num_neighbors=8) diff --git a/smac/optimizer/ei_optimization.py b/smac/optimizer/ei_optimization.py index 99e6d3167..11587ad88 100644 --- a/smac/optimizer/ei_optimization.py +++ b/smac/optimizer/ei_optimization.py @@ -12,6 +12,7 @@ Configuration, ConfigurationSpace, convert_configurations_to_array, + ForbiddenValueError, ) from smac.runhistory.runhistory import RunHistory from smac.stats.stats import Stats @@ -367,10 +368,15 @@ def _do_search( n = next(neighborhood_iterator) neighbors_generated[i] += 1 neighbors_for_i.append(n) + except ValueError as e: + # `neighborhood_iterator` raises `ValueError` with some probability when it reaches + # an invalid configuration. + self.logger.debug(e) + new_neighborhood[i] = True except StopIteration: - obtain_n[i] = len(neighbors_for_i) new_neighborhood[i] = True break + obtain_n[i] = len(neighbors_for_i) neighbors.extend(neighbors_for_i) if len(neighbors) != 0: @@ -398,18 +404,25 @@ def _do_search( # Found a better configuration if acq_val[acq_index] > acq_val_candidates[i]: - self.logger.debug( - "Local search %d: Switch to one of the neighbors (after %d configurations).", - i, - neighbors_looked_at[i], - ) - candidates[i] = neighbors[acq_index] - acq_val_candidates[i] = acq_val[acq_index] - new_neighborhood[i] = True - improved[i] = True - local_search_steps[i] += 1 - neighbors_w_equal_acq[i] = [] - obtain_n[i] = 1 + is_valid = False + try: + neighbors[acq_index].is_valid_configuration() + is_valid = True + except (ValueError, ForbiddenValueError) as e: + self.logger.debug("Local search %d: %s", i, e) + if is_valid: + self.logger.debug( + "Local search %d: Switch to one of the neighbors (after %d configurations).", + i, + neighbors_looked_at[i], + ) + candidates[i] = neighbors[acq_index] + acq_val_candidates[i] = acq_val[acq_index] + new_neighborhood[i] = True + improved[i] = True + local_search_steps[i] += 1 + neighbors_w_equal_acq[i] = [] + obtain_n[i] = 1 # Found an equally well performing configuration, keeping it for plateau walking elif acq_val[acq_index] == acq_val_candidates[i]: neighbors_w_equal_acq[i].append(neighbors[acq_index])