Handle invalid neighbors in local search gracefully (#773)

* Catch ValueError raised by neighborhood iterator

If the iterator returned by
`ConfigSpace.util.get_one_exchange_neighborhood`
reaches an invalid configuration,
it raises a `ValueError` with the probability 5 %.
The randomness is sampled using `np.random.random()`,
so it is non-deterministic.
See ConfigSpace/util.pyx:218.

This commit ensures that the exception is caught and handled gracefully.

* Consider only valid configurations in local search

In local search, only advance to a better-scoring neighbor
if that neighbor is a valid configuration.

Note that `neighborhood_iterator` may yield an invalid configuration.

smac/configspace/__init__.py

@@ -3,6 +3,7 @@
 from ConfigSpace import ConfigurationSpace, Configuration, Constant, \
     CategoricalHyperparameter, UniformFloatHyperparameter, \
     UniformIntegerHyperparameter, InCondition
+from ConfigSpace.exceptions import ForbiddenValueError
 from ConfigSpace.read_and_write import pcs, pcs_new, json
 from ConfigSpace.util import get_one_exchange_neighbourhood
 from smac.configspace.util import convert_configurations_to_array
@@ -22,7 +23,8 @@
            "pcs_new",
            "json",
            "get_one_exchange_neighbourhood",
-           "convert_configurations_to_array"
+           "convert_configurations_to_array",
+           "ForbiddenValueError"
            ]
 
 get_one_exchange_neighbourhood = partial(get_one_exchange_neighbourhood, stdev=0.05, num_neighbors=8)

smac/optimizer/ei_optimization.py

@@ -12,6 +12,7 @@
     Configuration,
     ConfigurationSpace,
     convert_configurations_to_array,
+    ForbiddenValueError,
 )
 from smac.runhistory.runhistory import RunHistory
 from smac.stats.stats import Stats
@@ -367,10 +368,15 @@ def _do_search(
                             n = next(neighborhood_iterator)
                             neighbors_generated[i] += 1
                             neighbors_for_i.append(n)
+                        except ValueError as e:
+                            # `neighborhood_iterator` raises `ValueError` with some probability when it reaches
+                            # an invalid configuration.
+                            self.logger.debug(e)
+                            new_neighborhood[i] = True
                         except StopIteration:
-                            obtain_n[i] = len(neighbors_for_i)
                             new_neighborhood[i] = True
                             break
+                    obtain_n[i] = len(neighbors_for_i)
                     neighbors.extend(neighbors_for_i)
 
             if len(neighbors) != 0:
@@ -398,18 +404,25 @@ def _do_search(
 
                             # Found a better configuration
                             if acq_val[acq_index] > acq_val_candidates[i]:
-                                self.logger.debug(
-                                    "Local search %d: Switch to one of the neighbors (after %d configurations).",
-                                    i,
-                                    neighbors_looked_at[i],
-                                )
-                                candidates[i] = neighbors[acq_index]
-                                acq_val_candidates[i] = acq_val[acq_index]
-                                new_neighborhood[i] = True
-                                improved[i] = True
-                                local_search_steps[i] += 1
-                                neighbors_w_equal_acq[i] = []
-                                obtain_n[i] = 1
+                                is_valid = False
+                                try:
+                                    neighbors[acq_index].is_valid_configuration()
+                                    is_valid = True
+                                except (ValueError, ForbiddenValueError) as e:
+                                    self.logger.debug("Local search %d: %s", i, e)
+                                if is_valid:
+                                    self.logger.debug(
+                                        "Local search %d: Switch to one of the neighbors (after %d configurations).",
+                                        i,
+                                        neighbors_looked_at[i],
+                                    )
+                                    candidates[i] = neighbors[acq_index]
+                                    acq_val_candidates[i] = acq_val[acq_index]
+                                    new_neighborhood[i] = True
+                                    improved[i] = True
+                                    local_search_steps[i] += 1
+                                    neighbors_w_equal_acq[i] = []
+                                    obtain_n[i] = 1
                             # Found an equally well performing configuration, keeping it for plateau walking
                             elif acq_val[acq_index] == acq_val_candidates[i]:
                                 neighbors_w_equal_acq[i].append(neighbors[acq_index])