From 0a452fbc649a8a2af63b2c34b522e5f9a6d86db0 Mon Sep 17 00:00:00 2001 From: Angel Ferran Pousa Date: Tue, 12 Mar 2024 11:22:39 +0100 Subject: [PATCH 1/9] Allow adding trials out of the range of the varying parameters --- optimas/generators/ax/service/base.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/optimas/generators/ax/service/base.py b/optimas/generators/ax/service/base.py index c69d90f1..a18864f5 100644 --- a/optimas/generators/ax/service/base.py +++ b/optimas/generators/ax/service/base.py @@ -16,6 +16,7 @@ GenerationStep, GenerationStrategy, ) +from ax import Arm from optimas.utils.other import update_object from optimas.core import ( @@ -158,7 +159,13 @@ def _tell(self, trials: List[Trial]) -> None: trial.varying_parameters, trial.parameter_values ): params[var.name] = value - _, trial_id = self._ax_client.attach_trial(params) + try: + _, trial_id = self._ax_client.attach_trial(params) + except ValueError: + trial = self._ax_client.experiment.new_trial() + trial.add_arm(Arm(parameters=params)) + trial.mark_running(no_runner_required=True) + trial_id = trial.index ax_trial = self._ax_client.get_trial(trial_id) # Since data was given externally, reduce number of From 3baa0a541bc1596490fb6980a119ad675cdfd4d3 Mon Sep 17 00:00:00 2001 From: Angel Ferran Pousa Date: Tue, 12 Mar 2024 12:41:52 +0100 Subject: [PATCH 2/9] Fix bug and improve implementation --- optimas/generators/ax/service/base.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/optimas/generators/ax/service/base.py b/optimas/generators/ax/service/base.py index a18864f5..0e170973 100644 --- a/optimas/generators/ax/service/base.py +++ b/optimas/generators/ax/service/base.py @@ -161,11 +161,19 @@ def _tell(self, trials: List[Trial]) -> None: params[var.name] = value try: _, trial_id = self._ax_client.attach_trial(params) - except ValueError: - trial = self._ax_client.experiment.new_trial() - trial.add_arm(Arm(parameters=params)) - trial.mark_running(no_runner_required=True) - trial_id = trial.index + except ValueError as error: + # Bypass checks from AxClient and manually add a trial + # outside of the search space. + if ( + "not a valid value" in str(error) + and self._fit_out_of_design + ): + ax_trial = self._ax_client.experiment.new_trial() + ax_trial.add_arm(Arm(parameters=params)) + ax_trial.mark_running(no_runner_required=True) + trial_id = ax_trial.index + else: + raise error ax_trial = self._ax_client.get_trial(trial_id) # Since data was given externally, reduce number of From a01fa4fa6c4a90459cf1006be16745cd2fed4cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Ferran=20Pousa?= Date: Fri, 15 Mar 2024 14:53:56 +0100 Subject: [PATCH 3/9] Add option to ignore trial --- optimas/core/trial.py | 32 ++++++++++++++++++++++++++ optimas/explorations/base.py | 3 +++ optimas/generators/ax/service/base.py | 33 ++++++++++++++++----------- optimas/generators/base.py | 24 ++++++++++++++----- 4 files changed, 73 insertions(+), 19 deletions(-) diff --git a/optimas/core/trial.py b/optimas/core/trial.py index d3815d6b..cc036a0c 100644 --- a/optimas/core/trial.py +++ b/optimas/core/trial.py @@ -66,6 +66,8 @@ def __init__( self._custom_parameters = ( [] if custom_parameters is None else custom_parameters ) + self._ignored = False + self._ignored_reason = None # Add custom parameters as trial attributes. for param in self._custom_parameters: @@ -127,6 +129,16 @@ def index(self) -> int: def index(self, value): self._index = value + @property + def ignored(self) -> bool: + """Get whether the trial is ignored by the generator.""" + return self._ignored + + @property + def ignored_reason(self) -> str: + """Get the reason why the trial is ignored by the generator.""" + return self._ignored_reason + @property def custom_parameters(self) -> List[TrialParameter]: """Get the list of custom trial parameters.""" @@ -152,6 +164,26 @@ def evaluated(self) -> bool: """Determine whether the trial has been evaluated.""" return self.completed or self.failed + def ignore(self, reason: str): + """Set trial as ignored. + + Parameters + ---------- + reason : str + The reason why the trial is ignored. + """ + # An alternative implementation of this would have been to add a new + # `IGNORED` trial status. However, this would have an issue: + # when adding old trials to an exploration, the original trial status + # could be overwritten by `IGNORED`, and this information would be lost + # for future explorations where this data is reused (for example, + # when using the `resume` option). + # With the current implementation, the value of `ignored` is controlled + # by (and only relevant for) the current exploration. It won't have + # any impact if the data is attached to a future exploration. + self._ignored = True + self._ignored_reason = reason + def mark_as(self, status) -> None: """Set trial status. diff --git a/optimas/explorations/base.py b/optimas/explorations/base.py index 7c5d2f65..4926017a 100644 --- a/optimas/explorations/base.py +++ b/optimas/explorations/base.py @@ -130,6 +130,9 @@ def history(self) -> pd.DataFrame: """Get the exploration history.""" history = convert_to_dataframe(self._libe_history.H) ordered_columns = ["trial_index", "trial_status"] + # For backward compatibility, add trial_ignored only if available. + if "trial_ignored" in history.columns.values.tolist(): + ordered_columns += ["trial_ignored"] ordered_columns += [p.name for p in self.generator.varying_parameters] ordered_columns += [p.name for p in self.generator.objectives] ordered_columns += [p.name for p in self.generator.analyzed_parameters] diff --git a/optimas/generators/ax/service/base.py b/optimas/generators/ax/service/base.py index 0e170973..d8031c03 100644 --- a/optimas/generators/ax/service/base.py +++ b/optimas/generators/ax/service/base.py @@ -164,24 +164,29 @@ def _tell(self, trials: List[Trial]) -> None: except ValueError as error: # Bypass checks from AxClient and manually add a trial # outside of the search space. - if ( - "not a valid value" in str(error) - and self._fit_out_of_design - ): - ax_trial = self._ax_client.experiment.new_trial() - ax_trial.add_arm(Arm(parameters=params)) - ax_trial.mark_running(no_runner_required=True) - trial_id = ax_trial.index + # https://github.com/facebook/Ax/issues/768#issuecomment-1036515242 + if "not a valid value" in str(error): + if self._fit_out_of_design: + ax_trial = self._ax_client.experiment.new_trial() + ax_trial.add_arm(Arm(parameters=params)) + ax_trial.mark_running(no_runner_required=True) + trial_id = ax_trial.index + else: + ignore_reason = ( + f"The parameters {params} are outside of the " + "range of the varying parameters. " + "Set `fit_out_of_design=True` if you want " + "the model to use these data." + ) + trial.ignore(reason=ignore_reason) + continue else: raise error ax_trial = self._ax_client.get_trial(trial_id) # Since data was given externally, reduce number of # initialization trials, but only if they have not failed. - if ( - trial.status != TrialStatus.FAILED - and not self._enforce_n_init - ): + if trial.completed and not self._enforce_n_init: generation_strategy = self._ax_client.generation_strategy current_step = generation_strategy.current_step # Reduce only if there are still Sobol trials left. @@ -191,7 +196,9 @@ def _tell(self, trials: List[Trial]) -> None: current_step.transition_criteria[0].threshold -= 1 generation_strategy._maybe_move_to_next_step() finally: - if trial.completed: + if trial.ignored: + continue + elif trial.completed: outcome_evals = {} # Add objective evaluations. for ev in trial.objective_evaluations: diff --git a/optimas/generators/base.py b/optimas/generators/base.py index fe62f2f6..354a9354 100644 --- a/optimas/generators/base.py +++ b/optimas/generators/base.py @@ -250,12 +250,19 @@ def tell( incorporating the evaluated trials. By default ``True``. """ + self._tell(trials) for trial in trials: if trial not in self._given_trials: self._add_external_evaluated_trial(trial) - self._tell(trials) - for trial in trials: - if not trial.failed: + + if trial.ignored: + # Check first if ignored, because it can be ignored + # and completed at the same time. + log_msg = ( + f"Trial {trial.index} ignored. " + f"Reason: {trial.ignored_reason}" + ) + elif trial.completed: log_msg = "Completed trial {} with objective(s) {}".format( trial.index, trial.objectives_as_dict() ) @@ -263,7 +270,7 @@ def tell( log_msg += " and analyzed parameter(s) {}".format( trial.analyzed_parameters_as_dict() ) - else: + elif trial.failed: log_msg = f"Failed to evaluate trial {trial.index}." logger.info(log_msg) if allow_saving_model and self._save_model: @@ -279,11 +286,15 @@ def incorporate_history(self, history: np.ndarray) -> None: """ # Keep only evaluations where the simulation finished successfully. - history = history[history["sim_ended"]] + history_ended = history[history["sim_ended"]] trials = self._create_trials_from_external_data( - history, ignore_unrecognized_parameters=True + history_ended, ignore_unrecognized_parameters=True ) self.tell(trials, allow_saving_model=False) + # Communicate to history array whether the trial has been ignored. + for trial in trials: + i = np.where(history["trial_index"] == trial.index)[0][0] + history["trial_ignored"][i] = trial.ignored def attach_trials( self, @@ -540,6 +551,7 @@ def get_gen_specs( [(var.name, var.dtype) for var in self._varying_parameters] + [("num_procs", int), ("num_gpus", int)] + [("trial_index", int)] + + [("trial_ignored", bool)] + [ (par.save_name, par.dtype) for par in self._custom_trial_parameters From d6ed0c450ff40daabd25eb852dc20b867798afbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Ferran=20Pousa?= Date: Fri, 15 Mar 2024 15:09:14 +0100 Subject: [PATCH 4/9] Add test --- tests/test_ax_generators.py | 94 ++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 13 deletions(-) diff --git a/tests/test_ax_generators.py b/tests/test_ax_generators.py index 8a6fd7d8..616e0428 100644 --- a/tests/test_ax_generators.py +++ b/tests/test_ax_generators.py @@ -410,6 +410,73 @@ def test_ax_single_fidelity_updated_params(): assert all(exploration.history["x0"][-3:] != -9) +def test_ax_single_fidelity_resume(): + """ + Test that an exploration with an AxService generator can resume + with an updated range of the varying parameters, even if some + old trials are out of the updated range. + """ + global trial_count + global trials_to_fail + trial_count = 0 + trials_to_fail = [] + + fit_out_of_design_vals = [False, True] + + for fit_out_of_design in fit_out_of_design_vals: + var1 = VaryingParameter("x0", 5.1, 6.0) + var2 = VaryingParameter("x1", -5.0, 15.0) + obj = Objective("f", minimize=False) + p1 = Parameter("p1") + + gen = AxSingleFidelityGenerator( + varying_parameters=[var1, var2], + objectives=[obj], + analyzed_parameters=[p1], + parameter_constraints=["x0 + x1 <= 10"], + outcome_constraints=["p1 <= 30"], + fit_out_of_design=fit_out_of_design, + ) + ev = FunctionEvaluator(function=eval_func_sf) + exploration = Exploration( + generator=gen, + evaluator=ev, + max_evals=20, + sim_workers=2, + exploration_dir_path="./tests_output/test_ax_single_fidelity", + libe_comms="local_threading", + resume=True, + ) + + # Get reference to original AxClient. + ax_client = gen._ax_client + + # Run exploration. + exploration.run(n_evals=1) + + if not fit_out_of_design: + # Check that no old evaluations were added + assert len(exploration.history) == 11 + assert all(exploration.history.trial_ignored.to_numpy()[:-1]) + # Check that the sobol step has not been skipped. + df = ax_client.get_trials_data_frame() + assert len(df) == 1 + assert df["generation_method"].to_numpy()[0] == "Sobol" + + else: + # Check that the old evaluations were added + assert len(exploration.history) == 12 + assert not all(exploration.history.trial_ignored.to_numpy()) + # Check that the sobol step has been skipped. + df = ax_client.get_trials_data_frame() + assert len(df) == 12 + assert df["generation_method"].to_numpy()[-1] == "GPEI" + + check_run_ax_service( + ax_client, gen, exploration, n_failed_expected=2 + ) + + def test_ax_multi_fidelity(): """Test that an exploration with a multifidelity generator runs""" @@ -769,16 +836,17 @@ def test_ax_service_init(): if __name__ == "__main__": - test_ax_single_fidelity() - test_ax_single_fidelity_int() - test_ax_single_fidelity_moo() - test_ax_single_fidelity_fb() - test_ax_single_fidelity_moo_fb() - test_ax_single_fidelity_updated_params() - test_ax_multi_fidelity() - test_ax_multitask() - test_ax_client() - test_ax_single_fidelity_with_history() - test_ax_multi_fidelity_with_history() - test_ax_multitask_with_history() - test_ax_service_init() + # test_ax_single_fidelity() + test_ax_single_fidelity_resume() + # test_ax_single_fidelity_int() + # test_ax_single_fidelity_moo() + # test_ax_single_fidelity_fb() + # test_ax_single_fidelity_moo_fb() + # test_ax_single_fidelity_updated_params() + # test_ax_multi_fidelity() + # test_ax_multitask() + # test_ax_client() + # test_ax_single_fidelity_with_history() + # test_ax_multi_fidelity_with_history() + # test_ax_multitask_with_history() + # test_ax_service_init() From 130d3fd84d00e4aeb29b5d1cb65d731608e3b622 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 14:17:59 +0000 Subject: [PATCH 5/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- optimas/core/trial.py | 4 ++-- optimas/generators/base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/optimas/core/trial.py b/optimas/core/trial.py index cc036a0c..6879cb32 100644 --- a/optimas/core/trial.py +++ b/optimas/core/trial.py @@ -133,7 +133,7 @@ def index(self, value): def ignored(self) -> bool: """Get whether the trial is ignored by the generator.""" return self._ignored - + @property def ignored_reason(self) -> str: """Get the reason why the trial is ignored by the generator.""" @@ -166,7 +166,7 @@ def evaluated(self) -> bool: def ignore(self, reason: str): """Set trial as ignored. - + Parameters ---------- reason : str diff --git a/optimas/generators/base.py b/optimas/generators/base.py index 354a9354..946f2b2b 100644 --- a/optimas/generators/base.py +++ b/optimas/generators/base.py @@ -254,7 +254,7 @@ def tell( for trial in trials: if trial not in self._given_trials: self._add_external_evaluated_trial(trial) - + if trial.ignored: # Check first if ignored, because it can be ignored # and completed at the same time. From ec05b5c28f11e74aa87e9cd07dc2a1a666a69828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Ferran=20Pousa?= Date: Fri, 15 Mar 2024 15:19:22 +0100 Subject: [PATCH 6/9] Uncomment tests --- tests/test_ax_generators.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_ax_generators.py b/tests/test_ax_generators.py index 616e0428..dd3e9907 100644 --- a/tests/test_ax_generators.py +++ b/tests/test_ax_generators.py @@ -836,17 +836,17 @@ def test_ax_service_init(): if __name__ == "__main__": - # test_ax_single_fidelity() + test_ax_single_fidelity() test_ax_single_fidelity_resume() - # test_ax_single_fidelity_int() - # test_ax_single_fidelity_moo() - # test_ax_single_fidelity_fb() - # test_ax_single_fidelity_moo_fb() - # test_ax_single_fidelity_updated_params() - # test_ax_multi_fidelity() - # test_ax_multitask() - # test_ax_client() - # test_ax_single_fidelity_with_history() - # test_ax_multi_fidelity_with_history() - # test_ax_multitask_with_history() - # test_ax_service_init() + test_ax_single_fidelity_int() + test_ax_single_fidelity_moo() + test_ax_single_fidelity_fb() + test_ax_single_fidelity_moo_fb() + test_ax_single_fidelity_updated_params() + test_ax_multi_fidelity() + test_ax_multitask() + test_ax_client() + test_ax_single_fidelity_with_history() + test_ax_multi_fidelity_with_history() + test_ax_multitask_with_history() + test_ax_service_init() From b10372dc8a8509b55303312288fbaf070313d5c1 Mon Sep 17 00:00:00 2001 From: delaossa Date: Thu, 6 Jun 2024 16:05:16 +0200 Subject: [PATCH 7/9] Allow `AxModelManager` to fit models with parameter values outside the design range. --- .../diagnostics/exploration_diagnostics.py | 11 +++++- optimas/utils/ax/ax_model_manager.py | 38 +++++++++++++++++-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/optimas/diagnostics/exploration_diagnostics.py b/optimas/diagnostics/exploration_diagnostics.py index 696cf6ee..69148e4e 100644 --- a/optimas/diagnostics/exploration_diagnostics.py +++ b/optimas/diagnostics/exploration_diagnostics.py @@ -946,7 +946,10 @@ def print_best_evaluations( print(best_evals[objective_names + varpar_names + anapar_names]) def build_gp_model( - self, parameter: str, minimize: Optional[bool] = None + self, + parameter: str, + minimize: Optional[bool] = None, + fit_out_of_design: Optional[bool] = False, ) -> AxModelManager: """Build a GP model of the specified parameter. @@ -960,6 +963,11 @@ def build_gp_model( Use it to indicate whether lower or higher values of the parameter are better. This is relevant, e.g. to determine the best point of the model. + fit_out_of_design : bool, optional + Whether to fit the surrogate model taking into account evaluations + outside of the range of the varying parameters. This can be useful + if the range of parameter has been reduced during the optimization. + By default, False. Returns ------- @@ -993,4 +1001,5 @@ def build_gp_model( source=self.history, varying_parameters=self.varying_parameters, objectives=[objective], + fit_out_of_design=fit_out_of_design, ) diff --git a/optimas/utils/ax/ax_model_manager.py b/optimas/utils/ax/ax_model_manager.py index 64118033..b73f0413 100644 --- a/optimas/utils/ax/ax_model_manager.py +++ b/optimas/utils/ax/ax_model_manager.py @@ -24,7 +24,7 @@ convert_optimas_to_ax_parameters, convert_optimas_to_ax_objectives, ) - + from ax import Arm ax_installed = True except ImportError: ax_installed = False @@ -53,6 +53,11 @@ class AxModelManager: parameters that were varied to scan the value of the objectives. The names and data of these parameters must be contained in the source ``DataFrame``. + fit_out_of_design : bool, optional + Whether to fit the surrogate model taking into account evaluations + outside of the range of the varying parameters. This can be useful + if the range of parameter has been reduced during the optimization. + By default, False. """ def __init__( @@ -60,6 +65,7 @@ def __init__( source: Union[AxClient, str, pd.DataFrame], varying_parameters: Optional[List[VaryingParameter]] = None, objectives: Optional[List[Objective]] = None, + fit_out_of_design: Optional[bool] = False, ) -> None: if not ax_installed: raise ImportError( @@ -72,7 +78,7 @@ def __init__( self.ax_client = AxClient.load_from_json_file(filepath=source) elif isinstance(source, pd.DataFrame): self.ax_client = self._build_ax_client_from_dataframe( - source, varying_parameters, objectives + source, varying_parameters, objectives, fit_out_of_design ) else: raise ValueError( @@ -93,6 +99,7 @@ def _build_ax_client_from_dataframe( df: pd.DataFrame, varying_parameters: List[VaryingParameter], objectives: List[Objective], + fit_out_of_design: Optional[bool] = False, ) -> AxClient: """Initialize the AxClient and the model using the given data. @@ -105,6 +112,9 @@ def _build_ax_client_from_dataframe( varying_parameters : list of `VaryingParameter`. List of parameters that were varied to scan the value of the objectives. + fit_out_of_design : bool, optional + Whether to fit the surrogate model taking into account evaluations + outside of the range of the varying parameters. """ # Define parameters for AxClient axparameters = convert_optimas_to_ax_parameters(varying_parameters) @@ -128,7 +138,29 @@ def _build_ax_client_from_dataframe( # Add trials from DataFrame for _, row in df.iterrows(): params = {vp.name: row[vp.name] for vp in varying_parameters} - _, trial_id = ax_client.attach_trial(params) + try: + _, trial_id = ax_client.attach_trial(params) + except ValueError as error: + # Bypass checks from AxClient and manually add a trial + # outside of the search space. + # https://github.com/facebook/Ax/issues/768#issuecomment-1036515242 + if "not a valid value" in str(error): + if fit_out_of_design: + ax_trial = ax_client.experiment.new_trial() + ax_trial.add_arm(Arm(parameters=params)) + ax_trial.mark_running(no_runner_required=True) + trial_id = ax_trial.index + else: + ignore_reason = ( + f"The parameters {params} are outside of the " + "range of the varying parameters. " + "Set `fit_out_of_design=True` if you want " + "the model to use these data." + ) + print(ignore_reason) + continue + else: + raise error data = {obj.name: (row[obj.name], np.nan) for obj in objectives} ax_client.complete_trial(trial_id, raw_data=data) return ax_client From c69519fa57a6f9ab640aa0008bad9e6eb223e3a9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 14:06:56 +0000 Subject: [PATCH 8/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- optimas/utils/ax/ax_model_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/optimas/utils/ax/ax_model_manager.py b/optimas/utils/ax/ax_model_manager.py index b73f0413..2e014ea1 100644 --- a/optimas/utils/ax/ax_model_manager.py +++ b/optimas/utils/ax/ax_model_manager.py @@ -25,6 +25,7 @@ convert_optimas_to_ax_objectives, ) from ax import Arm + ax_installed = True except ImportError: ax_installed = False From 5f3a1d5ee154f02ef84ecea4471e1b15a568290f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Ferran=20Pousa?= Date: Tue, 11 Jun 2024 19:11:03 +0200 Subject: [PATCH 9/9] Fix issue preventing transition from Sobol to BO step --- optimas/generators/ax/service/base.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/optimas/generators/ax/service/base.py b/optimas/generators/ax/service/base.py index f995544b..3d2bf42b 100644 --- a/optimas/generators/ax/service/base.py +++ b/optimas/generators/ax/service/base.py @@ -15,6 +15,7 @@ GenerationStep, GenerationStrategy, ) +from ax.modelbridge.transition_criterion import MaxTrials, MinTrials from ax import Arm from optimas.core import ( @@ -197,7 +198,13 @@ def _tell(self, trials: List[Trial]) -> None: current_step = generation_strategy.current_step # Reduce only if there are still Sobol trials left. if current_step.model == Models.SOBOL: - current_step.transition_criteria[0].threshold -= 1 + for tc in current_step.transition_criteria: + # Looping over all criterial makes sure we reduce + # the transition thresholds due to `_n_init` + # (i.e., max trials) and `min_trials_observed=1` ( + # i.e., min trials). + if isinstance(tc, (MinTrials, MaxTrials)): + tc.threshold -= 1 generation_strategy._maybe_move_to_next_step() finally: if trial.ignored: