From 4f88aaf76769c6af3b22efe01999680d41896639 Mon Sep 17 00:00:00 2001 From: Angel Ferran Pousa Date: Tue, 7 May 2024 14:23:54 +0200 Subject: [PATCH 1/4] Do not crash if `n_init` < `n_workers` --- optimas/generators/ax/service/base.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/optimas/generators/ax/service/base.py b/optimas/generators/ax/service/base.py index 98373995..8c97ed86 100644 --- a/optimas/generators/ax/service/base.py +++ b/optimas/generators/ax/service/base.py @@ -15,6 +15,7 @@ GenerationStep, GenerationStrategy, ) +from ax.exceptions.core import DataRequiredError from optimas.core import ( Objective, @@ -127,13 +128,22 @@ def __init__( def _ask(self, trials: List[Trial]) -> List[Trial]: """Fill in the parameter values of the requested trials.""" for trial in trials: - parameters, trial_id = self._ax_client.get_next_trial( - fixed_features=self._fixed_features - ) - trial.parameter_values = [ - parameters.get(var.name) for var in self._varying_parameters - ] - trial.ax_trial_id = trial_id + try: + parameters, trial_id = self._ax_client.get_next_trial( + fixed_features=self._fixed_features + ) + trial.parameter_values = [ + parameters.get(var.name) for var in self._varying_parameters + ] + trial.ax_trial_id = trial_id + except DataRequiredError: + # This exception is raised when a BO model is asked to generate + # a trial when no data is available. This can happen, e.g., if + # none of the Sobol trials has not yet been completed. + # In this case, we simply ignore the exception to indicate that + # the trial could not be generated(an empty trial is returned, + # which will be filtered out by `Generator.ask`. + pass return trials def _tell(self, trials: List[Trial]) -> None: From 1fee312b092b4c849ac96a460501ede0af1b0347 Mon Sep 17 00:00:00 2001 From: Angel Ferran Pousa Date: Tue, 7 May 2024 15:22:41 +0200 Subject: [PATCH 2/4] Use Ax's own features to support case where sim_workers > n_init --- optimas/generators/ax/service/base.py | 39 +++++++++++-------- .../generators/ax/service/multi_fidelity.py | 4 +- .../generators/ax/service/single_fidelity.py | 4 +- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/optimas/generators/ax/service/base.py b/optimas/generators/ax/service/base.py index 8c97ed86..c8fc3d19 100644 --- a/optimas/generators/ax/service/base.py +++ b/optimas/generators/ax/service/base.py @@ -128,22 +128,13 @@ def __init__( def _ask(self, trials: List[Trial]) -> List[Trial]: """Fill in the parameter values of the requested trials.""" for trial in trials: - try: - parameters, trial_id = self._ax_client.get_next_trial( - fixed_features=self._fixed_features - ) - trial.parameter_values = [ - parameters.get(var.name) for var in self._varying_parameters - ] - trial.ax_trial_id = trial_id - except DataRequiredError: - # This exception is raised when a BO model is asked to generate - # a trial when no data is available. This can happen, e.g., if - # none of the Sobol trials has not yet been completed. - # In this case, we simply ignore the exception to indicate that - # the trial could not be generated(an empty trial is returned, - # which will be filtered out by `Generator.ask`. - pass + parameters, trial_id = self._ax_client.get_next_trial( + fixed_features=self._fixed_features + ) + trial.parameter_values = [ + parameters.get(var.name) for var in self._varying_parameters + ] + trial.ax_trial_id = trial_id return trials def _tell(self, trials: List[Trial]) -> None: @@ -247,6 +238,22 @@ def _create_ax_objectives(self) -> Dict[str, ObjectiveProperties]: for obj in self.objectives: objectives[obj.name] = ObjectiveProperties(minimize=obj.minimize) return objectives + + def _create_sobol_step(self) -> GenerationStep: + """Create a Sobol generation step with `n_init` trials.""" + # Ensure that at least 1 trial is completed before moving onto the BO + # step, and keep generating Sobol trials until that happens, even if + # the number of Sobol trials exceeds `n_init`. + # Otherwise, if we move to the BO step before any trial is completed, + # the next `ask` would fail with a `DataRequiredError`. + # This also allows the generator to work well when + # `sim_workers` > `n_init`. + return GenerationStep( + model=Models.SOBOL, + num_trials=self._n_init, + min_trials_observed=1, + enforce_num_trials=False, + ) def _create_generation_steps( self, bo_model_kwargs: Dict diff --git a/optimas/generators/ax/service/multi_fidelity.py b/optimas/generators/ax/service/multi_fidelity.py index e8a1ae3b..d67a5e3b 100644 --- a/optimas/generators/ax/service/multi_fidelity.py +++ b/optimas/generators/ax/service/multi_fidelity.py @@ -117,9 +117,7 @@ def _create_generation_steps( steps = [] # Add Sobol initialization with `n_init` random trials. - steps.append( - GenerationStep(model=Models.SOBOL, num_trials=self._n_init) - ) + steps.append(self._create_sobol_step()) # Continue indefinitely with GPKG. steps.append( diff --git a/optimas/generators/ax/service/single_fidelity.py b/optimas/generators/ax/service/single_fidelity.py index c6ee31ec..3ec93dc3 100644 --- a/optimas/generators/ax/service/single_fidelity.py +++ b/optimas/generators/ax/service/single_fidelity.py @@ -156,9 +156,7 @@ def _create_generation_steps( steps = [] # Add Sobol initialization with `n_init` random trials. - steps.append( - GenerationStep(model=Models.SOBOL, num_trials=self._n_init) - ) + steps.append(self._create_sobol_step()) # Continue indefinitely with BO. steps.append( From 88b06c34cec02ae92d6f9ef0c88466393392db0c Mon Sep 17 00:00:00 2001 From: Angel Ferran Pousa Date: Tue, 7 May 2024 15:25:37 +0200 Subject: [PATCH 3/4] Update test --- tests/test_ax_generators.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_ax_generators.py b/tests/test_ax_generators.py index 9df605d7..c4cb2530 100644 --- a/tests/test_ax_generators.py +++ b/tests/test_ax_generators.py @@ -692,8 +692,8 @@ def test_ax_service_init(): var2 = VaryingParameter("x1", -5.0, 15.0) obj = Objective("f", minimize=False) - n_init = 4 - n_external = 6 + n_init = 2 + n_external = 4 for i in range(n_external): gen = AxSingleFidelityGenerator( @@ -704,7 +704,7 @@ def test_ax_service_init(): generator=gen, evaluator=ev, max_evals=6, - sim_workers=2, + sim_workers=3, # Test with sim_workers > n_init. exploration_dir_path=f"./tests_output/test_ax_service_init_{i}", ) @@ -753,7 +753,7 @@ def test_ax_service_init(): generator=gen, evaluator=ev, max_evals=15, - sim_workers=2, + sim_workers=3, # Test with sim_workers > n_init. exploration_dir_path="./tests_output/test_ax_service_init_enforce", ) From 249195a47b3da4344285ebeea15101975bae44a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 13:27:21 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- optimas/generators/ax/service/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimas/generators/ax/service/base.py b/optimas/generators/ax/service/base.py index c8fc3d19..6124f66e 100644 --- a/optimas/generators/ax/service/base.py +++ b/optimas/generators/ax/service/base.py @@ -238,7 +238,7 @@ def _create_ax_objectives(self) -> Dict[str, ObjectiveProperties]: for obj in self.objectives: objectives[obj.name] = ObjectiveProperties(minimize=obj.minimize) return objectives - + def _create_sobol_step(self) -> GenerationStep: """Create a Sobol generation step with `n_init` trials.""" # Ensure that at least 1 trial is completed before moving onto the BO