From 5cb2b22bef35a096983b60ad85fe582296126c5a Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Wed, 16 Feb 2022 22:19:19 +0100 Subject: [PATCH 01/22] Turn `early_stopping` into a Callable class --- python-package/lightgbm/callback.py | 228 ++++++++++++++-------------- 1 file changed, 111 insertions(+), 117 deletions(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 3eee0ba499d2..bc95c1353a6b 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -231,124 +231,118 @@ def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbos callback : callable The callback that activates early stopping. """ - best_score = [] - best_iter = [] - best_score_list: list = [] - cmp_op = [] - enabled = True - first_metric = '' - - def _init(env: CallbackEnv) -> None: - nonlocal best_score - nonlocal best_iter - nonlocal best_score_list - nonlocal cmp_op - nonlocal enabled - nonlocal first_metric - enabled = not any(env.params.get(boost_alias, "") == 'dart' for boost_alias - in _ConfigAliases.get("boosting")) - if not enabled: - _log_warning('Early stopping is not available in dart mode') - return - if not env.evaluation_result_list: - raise ValueError('For early stopping, ' - 'at least one dataset and eval metric is required for evaluation') - - if stopping_rounds <= 0: - raise ValueError("stopping_rounds should be greater than zero.") - - if verbose: - _log_info(f"Training until validation scores don't improve for {stopping_rounds} rounds") - - # reset storages - best_score = [] - best_iter = [] - best_score_list = [] - cmp_op = [] - first_metric = '' - - n_metrics = len(set(m[1] for m in env.evaluation_result_list)) - n_datasets = len(env.evaluation_result_list) // n_metrics - if isinstance(min_delta, list): - if not all(t >= 0 for t in min_delta): - raise ValueError('Values for early stopping min_delta must be non-negative.') - if len(min_delta) == 0: - if verbose: - _log_info('Disabling min_delta for early stopping.') - deltas = [0.0] * n_datasets * n_metrics - elif len(min_delta) == 1: - if verbose: - _log_info(f'Using {min_delta[0]} as min_delta for all metrics.') - deltas = min_delta * n_datasets * n_metrics - else: - if len(min_delta) != n_metrics: - raise ValueError('Must provide a single value for min_delta or as many as metrics.') - if first_metric_only and verbose: - _log_info(f'Using only {min_delta[0]} as early stopping min_delta.') - deltas = min_delta * n_datasets - else: - if min_delta < 0: - raise ValueError('Early stopping min_delta must be non-negative.') - if min_delta > 0 and n_metrics > 1 and not first_metric_only and verbose: - _log_info(f'Using {min_delta} as min_delta for all metrics.') - deltas = [min_delta] * n_datasets * n_metrics - - # split is needed for " " case (e.g. "train l1") - first_metric = env.evaluation_result_list[0][1].split(" ")[-1] - for eval_ret, delta in zip(env.evaluation_result_list, deltas): - best_iter.append(0) - best_score_list.append(None) - if eval_ret[3]: # greater is better - best_score.append(float('-inf')) - cmp_op.append(partial(_gt_delta, delta=delta)) + class _early_stopping_callback: + def __init__(self, stopping_rounds: int, first_metric_only: bool = False, verbose: bool = True, min_delta: Union[float, List[float]] = 0.0) -> None: + self.stopping_rounds = stopping_rounds + self.first_metric_only = first_metric_only + self.verbose = verbose + self.min_delta = min_delta + self.best_score = [] + self.best_iter = [] + self.best_score_list: list = [] + self.cmp_op = [] + self.enabled = True + self.first_metric = '' + + def _init(self, env: CallbackEnv) -> None: + self.enabled = not any(env.params.get(boost_alias, "") == 'dart' for boost_alias + in _ConfigAliases.get("boosting")) + if not self.enabled: + _log_warning('Early stopping is not available in dart mode') + return + if not env.evaluation_result_list: + raise ValueError('For early stopping, ' + 'at least one dataset and eval metric is required for evaluation') + + if self.stopping_rounds <= 0: + raise ValueError("stopping_rounds should be greater than zero.") + + if self.verbose: + _log_info(f"Training until validation scores don't improve for {self.stopping_rounds} rounds") + + # reset storages + self.best_score = [] + self.best_iter = [] + self.best_score_list = [] + self.cmp_op = [] + self.first_metric = '' + + n_metrics = len(set(m[1] for m in env.evaluation_result_list)) + n_datasets = len(env.evaluation_result_list) // n_metrics + if isinstance(self.min_delta, list): + if not all(t >= 0 for t in self.min_delta): + raise ValueError('Values for early stopping min_delta must be non-negative.') + if len(self.min_delta) == 0: + if self.verbose: + _log_info('Disabling min_delta for early stopping.') + deltas = [0.0] * n_datasets * n_metrics + elif len(self.min_delta) == 1: + if self.verbose: + _log_info(f'Using {self.min_delta[0]} as min_delta for all metrics.') + deltas = self.min_delta * n_datasets * n_metrics + else: + if len(self.min_delta) != n_metrics: + raise ValueError('Must provide a single value for min_delta or as many as metrics.') + if self.first_metric_only and self.verbose: + _log_info(f'Using only {self.min_delta[0]} as early stopping min_delta.') + deltas = self.min_delta * n_datasets else: - best_score.append(float('inf')) - cmp_op.append(partial(_lt_delta, delta=delta)) - - def _final_iteration_check(env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None: - nonlocal best_iter - nonlocal best_score_list - if env.iteration == env.end_iteration - 1: - if verbose: - best_score_str = '\t'.join([_format_eval_result(x) for x in best_score_list[i]]) - _log_info('Did not meet early stopping. ' - f'Best iteration is:\n[{best_iter[i] + 1}]\t{best_score_str}') - if first_metric_only: - _log_info(f"Evaluated only: {eval_name_splitted[-1]}") - raise EarlyStopException(best_iter[i], best_score_list[i]) + if self.min_delta < 0: + raise ValueError('Early stopping min_delta must be non-negative.') + if self.min_delta > 0 and n_metrics > 1 and not self.first_metric_only and self.verbose: + _log_info(f'Using {self.min_delta} as min_delta for all metrics.') + deltas = [self.min_delta] * n_datasets * n_metrics - def _callback(env: CallbackEnv) -> None: - nonlocal best_score - nonlocal best_iter - nonlocal best_score_list - nonlocal cmp_op - nonlocal enabled - nonlocal first_metric - if env.iteration == env.begin_iteration: - _init(env) - if not enabled: - return - for i in range(len(env.evaluation_result_list)): - score = env.evaluation_result_list[i][2] - if best_score_list[i] is None or cmp_op[i](score, best_score[i]): - best_score[i] = score - best_iter[i] = env.iteration - best_score_list[i] = env.evaluation_result_list # split is needed for " " case (e.g. "train l1") - eval_name_splitted = env.evaluation_result_list[i][1].split(" ") - if first_metric_only and first_metric != eval_name_splitted[-1]: - continue # use only the first metric for early stopping - if ((env.evaluation_result_list[i][0] == "cv_agg" and eval_name_splitted[0] == "train" - or env.evaluation_result_list[i][0] == env.model._train_data_name)): - _final_iteration_check(env, eval_name_splitted, i) - continue # train data for lgb.cv or sklearn wrapper (underlying lgb.train) - elif env.iteration - best_iter[i] >= stopping_rounds: - if verbose: - eval_result_str = '\t'.join([_format_eval_result(x) for x in best_score_list[i]]) - _log_info(f"Early stopping, best iteration is:\n[{best_iter[i] + 1}]\t{eval_result_str}") - if first_metric_only: + self.first_metric = env.evaluation_result_list[0][1].split(" ")[-1] + for eval_ret, delta in zip(env.evaluation_result_list, deltas): + print(eval_ret, delta) + self.best_iter.append(0) + self.best_score_list.append(None) + if eval_ret[3]: # greater is better + self.best_score.append(float('-inf')) + self.cmp_op.append(partial(_gt_delta, delta=delta)) + else: + self.best_score.append(float('inf')) + self.cmp_op.append(partial(_lt_delta, delta=delta)) + + def _final_iteration_check(self, env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None: + if env.iteration == env.end_iteration - 1: + if self.verbose: + best_score_str = '\t'.join([_format_eval_result(x) for x in best_score_list[i]]) + _log_info('Did not meet early stopping. ' + f'Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}') + if self.first_metric_only: _log_info(f"Evaluated only: {eval_name_splitted[-1]}") - raise EarlyStopException(best_iter[i], best_score_list[i]) - _final_iteration_check(env, eval_name_splitted, i) - _callback.order = 30 # type: ignore - return _callback + raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) + + def __call__(self, env: CallbackEnv) -> None: + if env.iteration == env.begin_iteration: + self._init(env) + if not self.enabled: + return + for i in range(len(env.evaluation_result_list)): + score = env.evaluation_result_list[i][2] + if self.best_score_list[i] is None or self.cmp_op[i](score, self.best_score[i]): + self.best_score[i] = score + self.best_iter[i] = env.iteration + self.best_score_list[i] = env.evaluation_result_list + # split is needed for " " case (e.g. "train l1") + eval_name_splitted = env.evaluation_result_list[i][1].split(" ") + if self.first_metric_only and self.first_metric != eval_name_splitted[-1]: + continue # use only the first metric for early stopping + if ((env.evaluation_result_list[i][0] == "cv_agg" and eval_name_splitted[0] == "train" + or env.evaluation_result_list[i][0] == env.model._train_data_name)): + self._final_iteration_check(env, eval_name_splitted, i) + continue # train data for lgb.cv or sklearn wrapper (underlying lgb.train) + elif env.iteration - self.best_iter[i] >= self.stopping_rounds: + if self.verbose: + eval_result_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) + _log_info(f"Early stopping, best iteration is:\n[{self.best_iter[i] + 1}]\t{eval_result_str}") + if self.first_metric_only: + _log_info(f"Evaluated only: {eval_name_splitted[-1]}") + raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) + self._final_iteration_check(env, eval_name_splitted, i) + + _early_stopping_callback.order = 30 # type: ignore + return _early_stopping_callback(stopping_rounds=stopping_rounds, first_metric_only=first_metric_only, verbose=verbose, min_delta=min_delta) From 6a0b91b4dee356acacaeb156d230e0425d687d2c Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 17 Feb 2022 10:09:37 +0100 Subject: [PATCH 02/22] Fix --- python-package/lightgbm/callback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index bc95c1353a6b..e66f4376deec 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -309,7 +309,7 @@ def _init(self, env: CallbackEnv) -> None: def _final_iteration_check(self, env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None: if env.iteration == env.end_iteration - 1: if self.verbose: - best_score_str = '\t'.join([_format_eval_result(x) for x in best_score_list[i]]) + best_score_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) _log_info('Did not meet early stopping. ' f'Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}') if self.first_metric_only: From 7ca8b557572446888cf793c0082d9a7efd1e29a7 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 17 Feb 2022 16:35:02 +0000 Subject: [PATCH 03/22] Lint --- python-package/lightgbm/callback.py | 73 ++++++++++++++++++----------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index e66f4376deec..6636eecce6f2 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -237,60 +237,69 @@ def __init__(self, stopping_rounds: int, first_metric_only: bool = False, verbos self.first_metric_only = first_metric_only self.verbose = verbose self.min_delta = min_delta + + self.enabled = True + + self._reset_storages() + + def _reset_storages(self) -> None: + # reset storages self.best_score = [] self.best_iter = [] - self.best_score_list: list = [] + self.best_score_list = [] self.cmp_op = [] - self.enabled = True self.first_metric = '' def _init(self, env: CallbackEnv) -> None: self.enabled = not any(env.params.get(boost_alias, "") == 'dart' for boost_alias - in _ConfigAliases.get("boosting")) + in _ConfigAliases.get("boosting")) if not self.enabled: _log_warning('Early stopping is not available in dart mode') return if not env.evaluation_result_list: raise ValueError('For early stopping, ' - 'at least one dataset and eval metric is required for evaluation') + 'at least one dataset and eval metric is required for evaluation') if self.stopping_rounds <= 0: - raise ValueError("stopping_rounds should be greater than zero.") + raise ValueError( + "stopping_rounds should be greater than zero.") if self.verbose: - _log_info(f"Training until validation scores don't improve for {self.stopping_rounds} rounds") + _log_info( + f"Training until validation scores don't improve for {self.stopping_rounds} rounds") - # reset storages - self.best_score = [] - self.best_iter = [] - self.best_score_list = [] - self.cmp_op = [] - self.first_metric = '' + self._reset_storages() n_metrics = len(set(m[1] for m in env.evaluation_result_list)) n_datasets = len(env.evaluation_result_list) // n_metrics if isinstance(self.min_delta, list): if not all(t >= 0 for t in self.min_delta): - raise ValueError('Values for early stopping min_delta must be non-negative.') + raise ValueError( + 'Values for early stopping min_delta must be non-negative.') if len(self.min_delta) == 0: if self.verbose: _log_info('Disabling min_delta for early stopping.') deltas = [0.0] * n_datasets * n_metrics elif len(self.min_delta) == 1: if self.verbose: - _log_info(f'Using {self.min_delta[0]} as min_delta for all metrics.') + _log_info( + f'Using {self.min_delta[0]} as min_delta for all metrics.') deltas = self.min_delta * n_datasets * n_metrics else: if len(self.min_delta) != n_metrics: - raise ValueError('Must provide a single value for min_delta or as many as metrics.') + raise ValueError( + 'Must provide a single value for min_delta or as many as metrics.') if self.first_metric_only and self.verbose: - _log_info(f'Using only {self.min_delta[0]} as early stopping min_delta.') + _log_info( + f'Using only {self.min_delta[0]} as early stopping min_delta.') deltas = self.min_delta * n_datasets else: if self.min_delta < 0: - raise ValueError('Early stopping min_delta must be non-negative.') + raise ValueError( + 'Early stopping min_delta must be non-negative.') if self.min_delta > 0 and n_metrics > 1 and not self.first_metric_only and self.verbose: - _log_info(f'Using {self.min_delta} as min_delta for all metrics.') + _log_info( + f'Using {self.min_delta} as min_delta for all metrics.') deltas = [self.min_delta] * n_datasets * n_metrics # split is needed for " " case (e.g. "train l1") @@ -309,12 +318,14 @@ def _init(self, env: CallbackEnv) -> None: def _final_iteration_check(self, env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None: if env.iteration == env.end_iteration - 1: if self.verbose: - best_score_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) + best_score_str = '\t'.join( + [_format_eval_result(x) for x in self.best_score_list[i]]) _log_info('Did not meet early stopping. ' - f'Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}') + f'Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}') if self.first_metric_only: _log_info(f"Evaluated only: {eval_name_splitted[-1]}") - raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) + raise EarlyStopException( + self.best_iter[i], self.best_score_list[i]) def __call__(self, env: CallbackEnv) -> None: if env.iteration == env.begin_iteration: @@ -328,20 +339,26 @@ def __call__(self, env: CallbackEnv) -> None: self.best_iter[i] = env.iteration self.best_score_list[i] = env.evaluation_result_list # split is needed for " " case (e.g. "train l1") - eval_name_splitted = env.evaluation_result_list[i][1].split(" ") + eval_name_splitted = env.evaluation_result_list[i][1].split( + " ") if self.first_metric_only and self.first_metric != eval_name_splitted[-1]: continue # use only the first metric for early stopping if ((env.evaluation_result_list[i][0] == "cv_agg" and eval_name_splitted[0] == "train" - or env.evaluation_result_list[i][0] == env.model._train_data_name)): + or env.evaluation_result_list[i][0] == env.model._train_data_name)): self._final_iteration_check(env, eval_name_splitted, i) - continue # train data for lgb.cv or sklearn wrapper (underlying lgb.train) + # train data for lgb.cv or sklearn wrapper (underlying lgb.train) + continue elif env.iteration - self.best_iter[i] >= self.stopping_rounds: if self.verbose: - eval_result_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) - _log_info(f"Early stopping, best iteration is:\n[{self.best_iter[i] + 1}]\t{eval_result_str}") + eval_result_str = '\t'.join( + [_format_eval_result(x) for x in self.best_score_list[i]]) + _log_info( + f"Early stopping, best iteration is:\n[{self.best_iter[i] + 1}]\t{eval_result_str}") if self.first_metric_only: - _log_info(f"Evaluated only: {eval_name_splitted[-1]}") - raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) + _log_info( + f"Evaluated only: {eval_name_splitted[-1]}") + raise EarlyStopException( + self.best_iter[i], self.best_score_list[i]) self._final_iteration_check(env, eval_name_splitted, i) _early_stopping_callback.order = 30 # type: ignore From e846975ac4f46031eb706a226c17d37bb4737456 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 17 Feb 2022 16:44:29 +0000 Subject: [PATCH 04/22] Remove print --- python-package/lightgbm/callback.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 6636eecce6f2..07369134a570 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -231,6 +231,7 @@ def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbos callback : callable The callback that activates early stopping. """ + class _early_stopping_callback: def __init__(self, stopping_rounds: int, first_metric_only: bool = False, verbose: bool = True, min_delta: Union[float, List[float]] = 0.0) -> None: self.stopping_rounds = stopping_rounds @@ -239,11 +240,9 @@ def __init__(self, stopping_rounds: int, first_metric_only: bool = False, verbos self.min_delta = min_delta self.enabled = True - self._reset_storages() def _reset_storages(self) -> None: - # reset storages self.best_score = [] self.best_iter = [] self.best_score_list = [] @@ -305,7 +304,6 @@ def _init(self, env: CallbackEnv) -> None: # split is needed for " " case (e.g. "train l1") self.first_metric = env.evaluation_result_list[0][1].split(" ")[-1] for eval_ret, delta in zip(env.evaluation_result_list, deltas): - print(eval_ret, delta) self.best_iter.append(0) self.best_score_list.append(None) if eval_ret[3]: # greater is better @@ -360,6 +358,5 @@ def __call__(self, env: CallbackEnv) -> None: raise EarlyStopException( self.best_iter[i], self.best_score_list[i]) self._final_iteration_check(env, eval_name_splitted, i) - _early_stopping_callback.order = 30 # type: ignore return _early_stopping_callback(stopping_rounds=stopping_rounds, first_metric_only=first_metric_only, verbose=verbose, min_delta=min_delta) From 9efb68de40f6b30ddb3ba7f41b9c4d75504ab92c Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 17 Feb 2022 17:16:38 +0000 Subject: [PATCH 05/22] Fix order --- python-package/lightgbm/callback.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 07369134a570..1ec76b2c3d11 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -234,6 +234,9 @@ def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbos class _early_stopping_callback: def __init__(self, stopping_rounds: int, first_metric_only: bool = False, verbose: bool = True, min_delta: Union[float, List[float]] = 0.0) -> None: + self.order = 30 + self.before_iteration = False + self.stopping_rounds = stopping_rounds self.first_metric_only = first_metric_only self.verbose = verbose @@ -358,5 +361,5 @@ def __call__(self, env: CallbackEnv) -> None: raise EarlyStopException( self.best_iter[i], self.best_score_list[i]) self._final_iteration_check(env, eval_name_splitted, i) - _early_stopping_callback.order = 30 # type: ignore + return _early_stopping_callback(stopping_rounds=stopping_rounds, first_metric_only=first_metric_only, verbose=verbose, min_delta=min_delta) From 993b175d68cc6d0b8ba7cf4a2ebeeee1e65dce16 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Fri, 18 Feb 2022 09:50:56 +0000 Subject: [PATCH 06/22] Revert "Lint" This reverts commit 7ca8b557572446888cf793c0082d9a7efd1e29a7. --- python-package/lightgbm/callback.py | 53 +++++++++++------------------ 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 1ec76b2c3d11..679d611c6771 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -248,7 +248,7 @@ def __init__(self, stopping_rounds: int, first_metric_only: bool = False, verbos def _reset_storages(self) -> None: self.best_score = [] self.best_iter = [] - self.best_score_list = [] + self.best_score_list: list = [] self.cmp_op = [] self.first_metric = '' @@ -263,45 +263,38 @@ def _init(self, env: CallbackEnv) -> None: 'at least one dataset and eval metric is required for evaluation') if self.stopping_rounds <= 0: - raise ValueError( - "stopping_rounds should be greater than zero.") + raise ValueError("stopping_rounds should be greater than zero.") if self.verbose: - _log_info( - f"Training until validation scores don't improve for {self.stopping_rounds} rounds") + _log_info(f"Training until validation scores don't improve for {self.stopping_rounds} rounds") + # reset storages self._reset_storages() n_metrics = len(set(m[1] for m in env.evaluation_result_list)) n_datasets = len(env.evaluation_result_list) // n_metrics if isinstance(self.min_delta, list): if not all(t >= 0 for t in self.min_delta): - raise ValueError( - 'Values for early stopping min_delta must be non-negative.') + raise ValueError('Values for early stopping min_delta must be non-negative.') if len(self.min_delta) == 0: if self.verbose: _log_info('Disabling min_delta for early stopping.') deltas = [0.0] * n_datasets * n_metrics elif len(self.min_delta) == 1: if self.verbose: - _log_info( - f'Using {self.min_delta[0]} as min_delta for all metrics.') + _log_info(f'Using {self.min_delta[0]} as min_delta for all metrics.') deltas = self.min_delta * n_datasets * n_metrics else: if len(self.min_delta) != n_metrics: - raise ValueError( - 'Must provide a single value for min_delta or as many as metrics.') + raise ValueError('Must provide a single value for min_delta or as many as metrics.') if self.first_metric_only and self.verbose: - _log_info( - f'Using only {self.min_delta[0]} as early stopping min_delta.') + _log_info(f'Using only {self.min_delta[0]} as early stopping min_delta.') deltas = self.min_delta * n_datasets else: if self.min_delta < 0: - raise ValueError( - 'Early stopping min_delta must be non-negative.') + raise ValueError('Early stopping min_delta must be non-negative.') if self.min_delta > 0 and n_metrics > 1 and not self.first_metric_only and self.verbose: - _log_info( - f'Using {self.min_delta} as min_delta for all metrics.') + _log_info(f'Using {self.min_delta} as min_delta for all metrics.') deltas = [self.min_delta] * n_datasets * n_metrics # split is needed for " " case (e.g. "train l1") @@ -319,14 +312,12 @@ def _init(self, env: CallbackEnv) -> None: def _final_iteration_check(self, env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None: if env.iteration == env.end_iteration - 1: if self.verbose: - best_score_str = '\t'.join( - [_format_eval_result(x) for x in self.best_score_list[i]]) + best_score_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) _log_info('Did not meet early stopping. ' f'Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}') if self.first_metric_only: _log_info(f"Evaluated only: {eval_name_splitted[-1]}") - raise EarlyStopException( - self.best_iter[i], self.best_score_list[i]) + raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) def __call__(self, env: CallbackEnv) -> None: if env.iteration == env.begin_iteration: @@ -340,26 +331,20 @@ def __call__(self, env: CallbackEnv) -> None: self.best_iter[i] = env.iteration self.best_score_list[i] = env.evaluation_result_list # split is needed for " " case (e.g. "train l1") - eval_name_splitted = env.evaluation_result_list[i][1].split( - " ") + eval_name_splitted = env.evaluation_result_list[i][1].split(" ") if self.first_metric_only and self.first_metric != eval_name_splitted[-1]: continue # use only the first metric for early stopping if ((env.evaluation_result_list[i][0] == "cv_agg" and eval_name_splitted[0] == "train" - or env.evaluation_result_list[i][0] == env.model._train_data_name)): + or env.evaluation_result_list[i][0] == env.model._train_data_name)): self._final_iteration_check(env, eval_name_splitted, i) - # train data for lgb.cv or sklearn wrapper (underlying lgb.train) - continue + continue # train data for lgb.cv or sklearn wrapper (underlying lgb.train) elif env.iteration - self.best_iter[i] >= self.stopping_rounds: if self.verbose: - eval_result_str = '\t'.join( - [_format_eval_result(x) for x in self.best_score_list[i]]) - _log_info( - f"Early stopping, best iteration is:\n[{self.best_iter[i] + 1}]\t{eval_result_str}") + eval_result_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) + _log_info(f"Early stopping, best iteration is:\n[{self.best_iter[i] + 1}]\t{eval_result_str}") if self.first_metric_only: - _log_info( - f"Evaluated only: {eval_name_splitted[-1]}") - raise EarlyStopException( - self.best_iter[i], self.best_score_list[i]) + _log_info(f"Evaluated only: {eval_name_splitted[-1]}") + raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) self._final_iteration_check(env, eval_name_splitted, i) return _early_stopping_callback(stopping_rounds=stopping_rounds, first_metric_only=first_metric_only, verbose=verbose, min_delta=min_delta) From 4e3fd0b0e0f4ccaef8d8e89c2134e1e5216bd9a4 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Fri, 18 Feb 2022 09:51:12 +0000 Subject: [PATCH 07/22] Apply suggestion from code review --- python-package/lightgbm/callback.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 679d611c6771..faa8d38f6346 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -233,7 +233,13 @@ def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbos """ class _early_stopping_callback: - def __init__(self, stopping_rounds: int, first_metric_only: bool = False, verbose: bool = True, min_delta: Union[float, List[float]] = 0.0) -> None: + def __init__( + self, + stopping_rounds: int, + first_metric_only: bool = False, + verbose: bool = True, + min_delta: Union[float, List[float]] = 0.0 + ) -> None: self.order = 30 self.before_iteration = False From 993d7413423cc2227492d709934f700166e75b74 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Fri, 18 Feb 2022 09:52:09 +0000 Subject: [PATCH 08/22] Nit --- python-package/lightgbm/callback.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index faa8d38f6346..11469efb275a 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -254,7 +254,7 @@ def __init__( def _reset_storages(self) -> None: self.best_score = [] self.best_iter = [] - self.best_score_list: list = [] + self.best_score_list = [] self.cmp_op = [] self.first_metric = '' @@ -274,7 +274,6 @@ def _init(self, env: CallbackEnv) -> None: if self.verbose: _log_info(f"Training until validation scores don't improve for {self.stopping_rounds} rounds") - # reset storages self._reset_storages() n_metrics = len(set(m[1] for m in env.evaluation_result_list)) From 644f03d050b1cb446be82a441b212ab0db02c0e9 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Fri, 18 Feb 2022 10:19:45 +0000 Subject: [PATCH 09/22] Lint --- python-package/lightgbm/callback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 11469efb275a..2c34a304f561 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -340,7 +340,7 @@ def __call__(self, env: CallbackEnv) -> None: if self.first_metric_only and self.first_metric != eval_name_splitted[-1]: continue # use only the first metric for early stopping if ((env.evaluation_result_list[i][0] == "cv_agg" and eval_name_splitted[0] == "train" - or env.evaluation_result_list[i][0] == env.model._train_data_name)): + or env.evaluation_result_list[i][0] == env.model._train_data_name)): self._final_iteration_check(env, eval_name_splitted, i) continue # train data for lgb.cv or sklearn wrapper (underlying lgb.train) elif env.iteration - self.best_iter[i] >= self.stopping_rounds: From 4ed874f82d71b8dcfaa5bce64bff9731154b823f Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 17:05:59 +0000 Subject: [PATCH 10/22] Move callable class outside the func for pickling --- python-package/lightgbm/callback.py | 242 ++++++++++++++-------------- 1 file changed, 123 insertions(+), 119 deletions(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 2c34a304f561..21e58d945374 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -199,6 +199,127 @@ def _callback(env: CallbackEnv) -> None: return _callback +class _EarlyStoppingCallback: + def __init__( + self, + stopping_rounds: int, + first_metric_only: bool = False, + verbose: bool = True, + min_delta: Union[float, List[float]] = 0.0 + ) -> None: + self.order = 30 + self.before_iteration = False + + self.stopping_rounds = stopping_rounds + self.first_metric_only = first_metric_only + self.verbose = verbose + self.min_delta = min_delta + + self.enabled = True + self._reset_storages() + + def _reset_storages(self) -> None: + self.best_score = [] + self.best_iter = [] + self.best_score_list = [] + self.cmp_op = [] + self.first_metric = '' + + def _init(self, env: CallbackEnv) -> None: + self.enabled = not any(env.params.get(boost_alias, "") == 'dart' for boost_alias + in _ConfigAliases.get("boosting")) + if not self.enabled: + _log_warning('Early stopping is not available in dart mode') + return + if not env.evaluation_result_list: + raise ValueError('For early stopping, ' + 'at least one dataset and eval metric is required for evaluation') + + if self.stopping_rounds <= 0: + raise ValueError("stopping_rounds should be greater than zero.") + + if self.verbose: + _log_info(f"Training until validation scores don't improve for {self.stopping_rounds} rounds") + + self._reset_storages() + + n_metrics = len(set(m[1] for m in env.evaluation_result_list)) + n_datasets = len(env.evaluation_result_list) // n_metrics + if isinstance(self.min_delta, list): + if not all(t >= 0 for t in self.min_delta): + raise ValueError('Values for early stopping min_delta must be non-negative.') + if len(self.min_delta) == 0: + if self.verbose: + _log_info('Disabling min_delta for early stopping.') + deltas = [0.0] * n_datasets * n_metrics + elif len(self.min_delta) == 1: + if self.verbose: + _log_info(f'Using {self.min_delta[0]} as min_delta for all metrics.') + deltas = self.min_delta * n_datasets * n_metrics + else: + if len(self.min_delta) != n_metrics: + raise ValueError('Must provide a single value for min_delta or as many as metrics.') + if self.first_metric_only and self.verbose: + _log_info(f'Using only {self.min_delta[0]} as early stopping min_delta.') + deltas = self.min_delta * n_datasets + else: + if self.min_delta < 0: + raise ValueError('Early stopping min_delta must be non-negative.') + if self.min_delta > 0 and n_metrics > 1 and not self.first_metric_only and self.verbose: + _log_info(f'Using {self.min_delta} as min_delta for all metrics.') + deltas = [self.min_delta] * n_datasets * n_metrics + + # split is needed for " " case (e.g. "train l1") + self.first_metric = env.evaluation_result_list[0][1].split(" ")[-1] + for eval_ret, delta in zip(env.evaluation_result_list, deltas): + self.best_iter.append(0) + self.best_score_list.append(None) + if eval_ret[3]: # greater is better + self.best_score.append(float('-inf')) + self.cmp_op.append(partial(_gt_delta, delta=delta)) + else: + self.best_score.append(float('inf')) + self.cmp_op.append(partial(_lt_delta, delta=delta)) + + def _final_iteration_check(self, env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None: + if env.iteration == env.end_iteration - 1: + if self.verbose: + best_score_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) + _log_info('Did not meet early stopping. ' + f'Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}') + if self.first_metric_only: + _log_info(f"Evaluated only: {eval_name_splitted[-1]}") + raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) + + def __call__(self, env: CallbackEnv) -> None: + if env.iteration == env.begin_iteration: + self._init(env) + if not self.enabled: + return + for i in range(len(env.evaluation_result_list)): + score = env.evaluation_result_list[i][2] + if self.best_score_list[i] is None or self.cmp_op[i](score, self.best_score[i]): + self.best_score[i] = score + self.best_iter[i] = env.iteration + self.best_score_list[i] = env.evaluation_result_list + # split is needed for " " case (e.g. "train l1") + eval_name_splitted = env.evaluation_result_list[i][1].split(" ") + if self.first_metric_only and self.first_metric != eval_name_splitted[-1]: + continue # use only the first metric for early stopping + if ((env.evaluation_result_list[i][0] == "cv_agg" and eval_name_splitted[0] == "train" + or env.evaluation_result_list[i][0] == env.model._train_data_name)): + self._final_iteration_check(env, eval_name_splitted, i) + continue # train data for lgb.cv or sklearn wrapper (underlying lgb.train) + elif env.iteration - self.best_iter[i] >= self.stopping_rounds: + if self.verbose: + eval_result_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) + _log_info(f"Early stopping, best iteration is:\n[{self.best_iter[i] + 1}]\t{eval_result_str}") + if self.first_metric_only: + _log_info(f"Evaluated only: {eval_name_splitted[-1]}") + raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) + self._final_iteration_check(env, eval_name_splitted, i) + + def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbose: bool = True, min_delta: Union[float, List[float]] = 0.0) -> Callable: """Create a callback that activates early stopping. @@ -232,124 +353,7 @@ def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbos The callback that activates early stopping. """ - class _early_stopping_callback: - def __init__( - self, - stopping_rounds: int, - first_metric_only: bool = False, - verbose: bool = True, - min_delta: Union[float, List[float]] = 0.0 - ) -> None: - self.order = 30 - self.before_iteration = False - - self.stopping_rounds = stopping_rounds - self.first_metric_only = first_metric_only - self.verbose = verbose - self.min_delta = min_delta - - self.enabled = True - self._reset_storages() - - def _reset_storages(self) -> None: - self.best_score = [] - self.best_iter = [] - self.best_score_list = [] - self.cmp_op = [] - self.first_metric = '' - - def _init(self, env: CallbackEnv) -> None: - self.enabled = not any(env.params.get(boost_alias, "") == 'dart' for boost_alias - in _ConfigAliases.get("boosting")) - if not self.enabled: - _log_warning('Early stopping is not available in dart mode') - return - if not env.evaluation_result_list: - raise ValueError('For early stopping, ' - 'at least one dataset and eval metric is required for evaluation') - - if self.stopping_rounds <= 0: - raise ValueError("stopping_rounds should be greater than zero.") + return _EarlyStoppingCallback(stopping_rounds=stopping_rounds, first_metric_only=first_metric_only, verbose=verbose, min_delta=min_delta) - if self.verbose: - _log_info(f"Training until validation scores don't improve for {self.stopping_rounds} rounds") - - self._reset_storages() - - n_metrics = len(set(m[1] for m in env.evaluation_result_list)) - n_datasets = len(env.evaluation_result_list) // n_metrics - if isinstance(self.min_delta, list): - if not all(t >= 0 for t in self.min_delta): - raise ValueError('Values for early stopping min_delta must be non-negative.') - if len(self.min_delta) == 0: - if self.verbose: - _log_info('Disabling min_delta for early stopping.') - deltas = [0.0] * n_datasets * n_metrics - elif len(self.min_delta) == 1: - if self.verbose: - _log_info(f'Using {self.min_delta[0]} as min_delta for all metrics.') - deltas = self.min_delta * n_datasets * n_metrics - else: - if len(self.min_delta) != n_metrics: - raise ValueError('Must provide a single value for min_delta or as many as metrics.') - if self.first_metric_only and self.verbose: - _log_info(f'Using only {self.min_delta[0]} as early stopping min_delta.') - deltas = self.min_delta * n_datasets - else: - if self.min_delta < 0: - raise ValueError('Early stopping min_delta must be non-negative.') - if self.min_delta > 0 and n_metrics > 1 and not self.first_metric_only and self.verbose: - _log_info(f'Using {self.min_delta} as min_delta for all metrics.') - deltas = [self.min_delta] * n_datasets * n_metrics - - # split is needed for " " case (e.g. "train l1") - self.first_metric = env.evaluation_result_list[0][1].split(" ")[-1] - for eval_ret, delta in zip(env.evaluation_result_list, deltas): - self.best_iter.append(0) - self.best_score_list.append(None) - if eval_ret[3]: # greater is better - self.best_score.append(float('-inf')) - self.cmp_op.append(partial(_gt_delta, delta=delta)) - else: - self.best_score.append(float('inf')) - self.cmp_op.append(partial(_lt_delta, delta=delta)) - - def _final_iteration_check(self, env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None: - if env.iteration == env.end_iteration - 1: - if self.verbose: - best_score_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) - _log_info('Did not meet early stopping. ' - f'Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}') - if self.first_metric_only: - _log_info(f"Evaluated only: {eval_name_splitted[-1]}") - raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) - - def __call__(self, env: CallbackEnv) -> None: - if env.iteration == env.begin_iteration: - self._init(env) - if not self.enabled: - return - for i in range(len(env.evaluation_result_list)): - score = env.evaluation_result_list[i][2] - if self.best_score_list[i] is None or self.cmp_op[i](score, self.best_score[i]): - self.best_score[i] = score - self.best_iter[i] = env.iteration - self.best_score_list[i] = env.evaluation_result_list - # split is needed for " " case (e.g. "train l1") - eval_name_splitted = env.evaluation_result_list[i][1].split(" ") - if self.first_metric_only and self.first_metric != eval_name_splitted[-1]: - continue # use only the first metric for early stopping - if ((env.evaluation_result_list[i][0] == "cv_agg" and eval_name_splitted[0] == "train" - or env.evaluation_result_list[i][0] == env.model._train_data_name)): - self._final_iteration_check(env, eval_name_splitted, i) - continue # train data for lgb.cv or sklearn wrapper (underlying lgb.train) - elif env.iteration - self.best_iter[i] >= self.stopping_rounds: - if self.verbose: - eval_result_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) - _log_info(f"Early stopping, best iteration is:\n[{self.best_iter[i] + 1}]\t{eval_result_str}") - if self.first_metric_only: - _log_info(f"Evaluated only: {eval_name_splitted[-1]}") - raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) - self._final_iteration_check(env, eval_name_splitted, i) - return _early_stopping_callback(stopping_rounds=stopping_rounds, first_metric_only=first_metric_only, verbose=verbose, min_delta=min_delta) +_EarlyStoppingCallback.__doc__ = early_stopping.__doc__ From cc299b28335a97797b4c6e91d8f3c48079ab42d1 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 17:06:35 +0000 Subject: [PATCH 11/22] Move _pickle and _unpickle to tests utils --- tests/python_package_test/test_dask.py | 47 +++++--------------------- tests/python_package_test/utils.py | 29 ++++++++++++++++ 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/python_package_test/test_dask.py b/tests/python_package_test/test_dask.py index a7632be0aa94..74cac88e3e26 100644 --- a/tests/python_package_test/test_dask.py +++ b/tests/python_package_test/test_dask.py @@ -2,7 +2,6 @@ """Tests for lightgbm.dask module""" import inspect -import pickle import random import socket from itertools import groupby @@ -22,10 +21,8 @@ if not lgb.compat.DASK_INSTALLED: pytest.skip('Dask is not installed', allow_module_level=True) -import cloudpickle import dask.array as da import dask.dataframe as dd -import joblib import numpy as np import pandas as pd import sklearn.utils.estimator_checks as sklearn_checks @@ -35,7 +32,7 @@ from scipy.stats import spearmanr from sklearn.datasets import make_blobs, make_regression -from .utils import make_ranking +from .utils import make_ranking, pickle_obj, unpickle_obj tasks = ['binary-classification', 'multiclass-classification', 'regression', 'ranking'] distributed_training_algorithms = ['data', 'voting'] @@ -232,32 +229,6 @@ def _constant_metric(y_true, y_pred): return metric_name, value, is_higher_better -def _pickle(obj, filepath, serializer): - if serializer == 'pickle': - with open(filepath, 'wb') as f: - pickle.dump(obj, f) - elif serializer == 'joblib': - joblib.dump(obj, filepath) - elif serializer == 'cloudpickle': - with open(filepath, 'wb') as f: - cloudpickle.dump(obj, f) - else: - raise ValueError(f'Unrecognized serializer type: {serializer}') - - -def _unpickle(filepath, serializer): - if serializer == 'pickle': - with open(filepath, 'rb') as f: - return pickle.load(f) - elif serializer == 'joblib': - return joblib.load(filepath) - elif serializer == 'cloudpickle': - with open(filepath, 'rb') as f: - return cloudpickle.load(f) - else: - raise ValueError(f'Unrecognized serializer type: {serializer}') - - def _objective_least_squares(y_true, y_pred): grad = y_pred - y_true hess = np.ones(len(y_true)) @@ -1358,23 +1329,23 @@ def test_model_and_local_version_are_picklable_whether_or_not_client_set_explici assert getattr(local_model, "client", None) is None tmp_file = tmp_path / "model-1.pkl" - _pickle( + pickle_obj( obj=dask_model, filepath=tmp_file, serializer=serializer ) - model_from_disk = _unpickle( + model_from_disk = unpickle_obj( filepath=tmp_file, serializer=serializer ) local_tmp_file = tmp_path / "local-model-1.pkl" - _pickle( + pickle_obj( obj=local_model, filepath=local_tmp_file, serializer=serializer ) - local_model_from_disk = _unpickle( + local_model_from_disk = unpickle_obj( filepath=local_tmp_file, serializer=serializer ) @@ -1414,23 +1385,23 @@ def test_model_and_local_version_are_picklable_whether_or_not_client_set_explici local_model.client_ tmp_file2 = tmp_path / "model-2.pkl" - _pickle( + pickle_obj( obj=dask_model, filepath=tmp_file2, serializer=serializer ) - fitted_model_from_disk = _unpickle( + fitted_model_from_disk = unpickle_obj( filepath=tmp_file2, serializer=serializer ) local_tmp_file2 = tmp_path / "local-model-2.pkl" - _pickle( + pickle_obj( obj=local_model, filepath=local_tmp_file2, serializer=serializer ) - local_fitted_model_from_disk = _unpickle( + local_fitted_model_from_disk = unpickle_obj( filepath=local_tmp_file2, serializer=serializer ) diff --git a/tests/python_package_test/utils.py b/tests/python_package_test/utils.py index 21d5f2cd542a..48abb4ab858b 100644 --- a/tests/python_package_test/utils.py +++ b/tests/python_package_test/utils.py @@ -1,6 +1,9 @@ # coding: utf-8 from functools import lru_cache +import pickle +import cloudpickle +import joblib import numpy as np import sklearn.datasets from sklearn.utils import check_random_state @@ -114,3 +117,29 @@ def make_ranking(n_samples=100, n_features=20, n_informative=5, gmax=2, @lru_cache(maxsize=None) def make_synthetic_regression(n_samples=100): return sklearn.datasets.make_regression(n_samples, n_features=4, n_informative=2, random_state=42) + + +def pickle_obj(obj, filepath, serializer): + if serializer == 'pickle': + with open(filepath, 'wb') as f: + pickle.dump(obj, f) + elif serializer == 'joblib': + joblib.dump(obj, filepath) + elif serializer == 'cloudpickle': + with open(filepath, 'wb') as f: + cloudpickle.dump(obj, f) + else: + raise ValueError(f'Unrecognized serializer type: {serializer}') + + +def unpickle_obj(filepath, serializer): + if serializer == 'pickle': + with open(filepath, 'rb') as f: + return pickle.load(f) + elif serializer == 'joblib': + return joblib.load(filepath) + elif serializer == 'cloudpickle': + with open(filepath, 'rb') as f: + return cloudpickle.load(f) + else: + raise ValueError(f'Unrecognized serializer type: {serializer}') From 72bceb434bbbff991d56a0c24272393b6fb762ea Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 17:06:55 +0000 Subject: [PATCH 12/22] Add early stopping callback picklability test --- tests/python_package_test/test_engine.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 66be939a7867..bdf490d0cfd6 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -18,7 +18,7 @@ import lightgbm as lgb -from .utils import load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression +from .utils import load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression, pickle_obj, unpickle_obj decreasing_generator = itertools.count(0, -1) @@ -3291,3 +3291,18 @@ def test_record_evaluation_with_cv(train_metric): np.testing.assert_allclose( cv_hist[key], eval_result[dataset][f'{metric}-{agg}'] ) + +@pytest.mark.parametrize('serializer', ["pickle", "joblib", "cloudpickle"]) +def test_early_stopping_callback_is_picklable(serializer, tmp_path): + callback = lgb.early_stopping(stopping_rounds=5) + tmp_file = tmp_path / "early_stopping.pkl" + pickle_obj( + obj=callback, + filepath=tmp_file, + serializer=serializer + ) + callback_from_disk = unpickle_obj( + filepath=tmp_file, + serializer=serializer + ) + assert callback.stopping_rounds == callback_from_disk.stopping_rounds From 314d43b0ae6e412df322ab5483fe93764d3a9c8f Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 17:13:27 +0000 Subject: [PATCH 13/22] Nit --- tests/python_package_test/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/utils.py b/tests/python_package_test/utils.py index 37e539f832a8..16534d924689 100644 --- a/tests/python_package_test/utils.py +++ b/tests/python_package_test/utils.py @@ -159,4 +159,4 @@ def unpickle_obj(filepath, serializer): with open(filepath, 'rb') as f: return cloudpickle.load(f) else: - raise ValueError(f'Unrecognized serializer type: {serializer}') \ No newline at end of file + raise ValueError(f'Unrecognized serializer type: {serializer}') From 7eff42f722cca95972d497a58d1c1104815d6387 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 17:23:38 +0000 Subject: [PATCH 14/22] Fix --- tests/python_package_test/test_engine.py | 73 +++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index cc934ff6d538..a4e75c0f8316 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -3307,8 +3307,77 @@ def test_pandas_with_numpy_regular_dtypes(): 'x2': rng.randint(1, 3, n_samples), 'x3': 10 * rng.randint(1, 3, n_samples), 'x4': 100 * rng.randint(1, 3, n_samples), - - + }) + df = df.astype(np.float64) + y = df['x1'] * (df['x2'] + df['x3'] + df['x4']) + ds = lgb.Dataset(df, y) + params = {'objective': 'l2', 'num_leaves': 31, 'min_child_samples': 1} + bst = lgb.train(params, ds, num_boost_round=5) + preds = bst.predict(df) + + # test all features were used + assert bst.trees_to_dataframe()['split_feature'].nunique() == df.shape[1] + # test the score is better than predicting the mean + baseline = np.full_like(y, y.mean()) + assert mean_squared_error(y, preds) < mean_squared_error(y, baseline) + + # test all predictions are equal using different input dtypes + for target_dtypes in [uints, ints, bool_and_floats]: + df2 = df.astype({f'x{i}': dtype for i, dtype in enumerate(target_dtypes, start=1)}) + assert df2.dtypes.tolist() == target_dtypes + ds2 = lgb.Dataset(df2, y) + bst2 = lgb.train(params, ds2, num_boost_round=5) + preds2 = bst2.predict(df2) + np.testing.assert_allclose(preds, preds2) + + +def test_pandas_nullable_dtypes(): + pd = pytest.importorskip('pandas') + rng = np.random.RandomState(0) + df = pd.DataFrame({ + 'x1': rng.randint(1, 3, size=100), + 'x2': np.linspace(-1, 1, 100), + 'x3': pd.arrays.SparseArray(rng.randint(0, 11, size=100)), + 'x4': rng.rand(100) < 0.5, + }) + # introduce some missing values + df.loc[1, 'x1'] = np.nan + df.loc[2, 'x2'] = np.nan + df.loc[3, 'x4'] = np.nan + # the previous line turns x3 into object dtype in recent versions of pandas + df['x4'] = df['x4'].astype(np.float64) + y = df['x1'] * df['x2'] + df['x3'] * (1 + df['x4']) + y = y.fillna(0) + + # train with regular dtypes + params = {'objective': 'l2', 'num_leaves': 31, 'min_child_samples': 1} + ds = lgb.Dataset(df, y) + bst = lgb.train(params, ds, num_boost_round=5) + preds = bst.predict(df) + + # convert to nullable dtypes + df2 = df.copy() + df2['x1'] = df2['x1'].astype('Int32') + df2['x2'] = df2['x2'].astype('Float64') + df2['x4'] = df2['x4'].astype('boolean') + + # test training succeeds + ds_nullable_dtypes = lgb.Dataset(df2, y) + bst_nullable_dtypes = lgb.train(params, ds_nullable_dtypes, num_boost_round=5) + preds_nullable_dtypes = bst_nullable_dtypes.predict(df2) + + trees_df = bst_nullable_dtypes.trees_to_dataframe() + # test all features were used + assert trees_df['split_feature'].nunique() == df.shape[1] + # test the score is better than predicting the mean + baseline = np.full_like(y, y.mean()) + assert mean_squared_error(y, preds) < mean_squared_error(y, baseline) + + # test equal predictions + np.testing.assert_allclose(preds, preds_nullable_dtypes) + + +@pytest.mark.parametrize('serializer', ["pickle", "joblib", "cloudpickle"]) def test_early_stopping_callback_is_picklable(serializer, tmp_path): callback = lgb.early_stopping(stopping_rounds=5) tmp_file = tmp_path / "early_stopping.pkl" From 3c9adfab85e1864ac507300734d4c631c885b1df Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 17:25:07 +0000 Subject: [PATCH 15/22] Lint --- python-package/lightgbm/callback.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 21e58d945374..873614b51966 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -227,13 +227,13 @@ def _reset_storages(self) -> None: def _init(self, env: CallbackEnv) -> None: self.enabled = not any(env.params.get(boost_alias, "") == 'dart' for boost_alias - in _ConfigAliases.get("boosting")) + in _ConfigAliases.get("boosting")) if not self.enabled: _log_warning('Early stopping is not available in dart mode') return if not env.evaluation_result_list: raise ValueError('For early stopping, ' - 'at least one dataset and eval metric is required for evaluation') + 'at least one dataset and eval metric is required for evaluation') if self.stopping_rounds <= 0: raise ValueError("stopping_rounds should be greater than zero.") @@ -286,7 +286,7 @@ def _final_iteration_check(self, env: CallbackEnv, eval_name_splitted: List[str] if self.verbose: best_score_str = '\t'.join([_format_eval_result(x) for x in self.best_score_list[i]]) _log_info('Did not meet early stopping. ' - f'Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}') + f'Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}') if self.first_metric_only: _log_info(f"Evaluated only: {eval_name_splitted[-1]}") raise EarlyStopException(self.best_iter[i], self.best_score_list[i]) From e271bdc24bd393c2d43b89dd8f0218fc662c5d8d Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 17:25:31 +0000 Subject: [PATCH 16/22] Improve type hint --- python-package/lightgbm/callback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 873614b51966..753c97528e43 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -320,7 +320,7 @@ def __call__(self, env: CallbackEnv) -> None: self._final_iteration_check(env, eval_name_splitted, i) -def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbose: bool = True, min_delta: Union[float, List[float]] = 0.0) -> Callable: +def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbose: bool = True, min_delta: Union[float, List[float]] = 0.0) -> _EarlyStoppingCallback: """Create a callback that activates early stopping. Activates early stopping. From e2130953b1edec29a875146304cb82c7eb25035d Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 20:51:09 +0000 Subject: [PATCH 17/22] Lint --- python-package/lightgbm/callback.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 753c97528e43..5446aacc8ae1 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -352,7 +352,6 @@ def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbos callback : callable The callback that activates early stopping. """ - return _EarlyStoppingCallback(stopping_rounds=stopping_rounds, first_metric_only=first_metric_only, verbose=verbose, min_delta=min_delta) From 20fc28262f347cd15a7ff39d9d862a1afcacfb47 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 22:38:45 +0000 Subject: [PATCH 18/22] Lint --- tests/python_package_test/test_engine.py | 3 ++- tests/python_package_test/utils.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index a4e75c0f8316..a6f3a0b70bd6 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -18,7 +18,8 @@ import lightgbm as lgb -from .utils import load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression, pickle_obj, unpickle_obj +from .utils import (load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression, pickle_obj, + unpickle_obj) decreasing_generator = itertools.count(0, -1) diff --git a/tests/python_package_test/utils.py b/tests/python_package_test/utils.py index 16534d924689..63950d471608 100644 --- a/tests/python_package_test/utils.py +++ b/tests/python_package_test/utils.py @@ -1,6 +1,6 @@ # coding: utf-8 -from functools import lru_cache import pickle +from functools import lru_cache import cloudpickle import joblib From 04a60d3f77e5d0476ff4f04d7d0e1ccc4fdb156f Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 24 Feb 2022 23:33:54 +0000 Subject: [PATCH 19/22] Add cloudpickle to test_windows --- .ci/test_windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index 1273d3713350..c59668fdd468 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -50,7 +50,7 @@ if ($env:TASK -eq "swig") { Exit 0 } -conda install -q -y -n $env:CONDA_ENV joblib matplotlib numpy pandas psutil pytest scikit-learn scipy ; Check-Output $? +conda install -q -y -n $env:CONDA_ENV cloudpickle joblib matplotlib numpy pandas psutil pytest scikit-learn scipy ; Check-Output $? # python-graphviz has to be installed separately to prevent conda from downgrading to pypy conda install -q -y -n $env:CONDA_ENV python-graphviz ; Check-Output $? From be20916b3604599e3036bcc5b056a20efdb792f4 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Wed, 2 Mar 2022 21:21:47 +0100 Subject: [PATCH 20/22] Update tests/python_package_test/test_engine.py --- tests/python_package_test/test_engine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 3b1bb81ac7af..d2ab30933caa 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -21,7 +21,6 @@ from .utils import (load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression, pickle_obj, unpickle_obj, sklearn_multiclass_custom_objective, softmax) - decreasing_generator = itertools.count(0, -1) From 0209eee355e93f374500e4066fe0393b6c5a9d2f Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Wed, 2 Mar 2022 20:27:07 +0000 Subject: [PATCH 21/22] Fix --- tests/python_package_test/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index d2ab30933caa..baccbc2305c1 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -19,7 +19,7 @@ import lightgbm as lgb from .utils import (load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression, pickle_obj, - unpickle_obj, sklearn_multiclass_custom_objective, softmax) + sklearn_multiclass_custom_objective, softmax, unpickle_obj) decreasing_generator = itertools.count(0, -1) From be04424eff919d3968d9a9dc07aac7b7b0965488 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 3 Mar 2022 17:54:42 +0000 Subject: [PATCH 22/22] Apply suggestions from code review --- python-package/lightgbm/callback.py | 25 ++++++++++------------ tests/python_package_test/test_callback.py | 22 +++++++++++++++++++ tests/python_package_test/test_engine.py | 20 ++--------------- 3 files changed, 35 insertions(+), 32 deletions(-) create mode 100644 tests/python_package_test/test_callback.py diff --git a/python-package/lightgbm/callback.py b/python-package/lightgbm/callback.py index 5446aacc8ae1..2fc301b0e509 100644 --- a/python-package/lightgbm/callback.py +++ b/python-package/lightgbm/callback.py @@ -12,14 +12,6 @@ ] -def _gt_delta(curr_score: float, best_score: float, delta: float) -> bool: - return curr_score > best_score + delta - - -def _lt_delta(curr_score: float, best_score: float, delta: float) -> bool: - return curr_score < best_score - delta - - class EarlyStopException(Exception): """Exception of early stopping.""" @@ -200,6 +192,8 @@ def _callback(env: CallbackEnv) -> None: class _EarlyStoppingCallback: + """Internal early stopping callable class.""" + def __init__( self, stopping_rounds: int, @@ -225,6 +219,12 @@ def _reset_storages(self) -> None: self.cmp_op = [] self.first_metric = '' + def _gt_delta(self, curr_score: float, best_score: float, delta: float) -> bool: + return curr_score > best_score + delta + + def _lt_delta(self, curr_score: float, best_score: float, delta: float) -> bool: + return curr_score < best_score - delta + def _init(self, env: CallbackEnv) -> None: self.enabled = not any(env.params.get(boost_alias, "") == 'dart' for boost_alias in _ConfigAliases.get("boosting")) @@ -276,10 +276,10 @@ def _init(self, env: CallbackEnv) -> None: self.best_score_list.append(None) if eval_ret[3]: # greater is better self.best_score.append(float('-inf')) - self.cmp_op.append(partial(_gt_delta, delta=delta)) + self.cmp_op.append(partial(self._gt_delta, delta=delta)) else: self.best_score.append(float('inf')) - self.cmp_op.append(partial(_lt_delta, delta=delta)) + self.cmp_op.append(partial(self._lt_delta, delta=delta)) def _final_iteration_check(self, env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None: if env.iteration == env.end_iteration - 1: @@ -349,10 +349,7 @@ def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbos Returns ------- - callback : callable + callback : _EarlyStoppingCallback The callback that activates early stopping. """ return _EarlyStoppingCallback(stopping_rounds=stopping_rounds, first_metric_only=first_metric_only, verbose=verbose, min_delta=min_delta) - - -_EarlyStoppingCallback.__doc__ = early_stopping.__doc__ diff --git a/tests/python_package_test/test_callback.py b/tests/python_package_test/test_callback.py new file mode 100644 index 000000000000..0f339aa3a53e --- /dev/null +++ b/tests/python_package_test/test_callback.py @@ -0,0 +1,22 @@ +# coding: utf-8 +import pytest + +import lightgbm as lgb + +from .utils import pickle_obj, unpickle_obj + + +@pytest.mark.parametrize('serializer', ["pickle", "joblib", "cloudpickle"]) +def test_early_stopping_callback_is_picklable(serializer, tmp_path): + callback = lgb.early_stopping(stopping_rounds=5) + tmp_file = tmp_path / "early_stopping.pkl" + pickle_obj( + obj=callback, + filepath=tmp_file, + serializer=serializer + ) + callback_from_disk = unpickle_obj( + filepath=tmp_file, + serializer=serializer + ) + assert callback.stopping_rounds == callback_from_disk.stopping_rounds diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index baccbc2305c1..df0a8c407dc0 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -18,8 +18,8 @@ import lightgbm as lgb -from .utils import (load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression, pickle_obj, - sklearn_multiclass_custom_objective, softmax, unpickle_obj) +from .utils import (load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression, + sklearn_multiclass_custom_objective, softmax) decreasing_generator = itertools.count(0, -1) @@ -3424,19 +3424,3 @@ def test_pandas_nullable_dtypes(): # test equal predictions np.testing.assert_allclose(preds, preds_nullable_dtypes) - - -@pytest.mark.parametrize('serializer', ["pickle", "joblib", "cloudpickle"]) -def test_early_stopping_callback_is_picklable(serializer, tmp_path): - callback = lgb.early_stopping(stopping_rounds=5) - tmp_file = tmp_path / "early_stopping.pkl" - pickle_obj( - obj=callback, - filepath=tmp_file, - serializer=serializer - ) - callback_from_disk = unpickle_obj( - filepath=tmp_file, - serializer=serializer - ) - assert callback.stopping_rounds == callback_from_disk.stopping_rounds