From b6c3dd0c117e494b9832dcb2f6a26f3bc0928188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Wed, 17 Jul 2024 13:51:33 +0200 Subject: [PATCH 01/14] Add options for storing --- metalearners/grid_search.py | 55 ++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index cc9c732..fdd8609 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: BSD-3-Clause import time -from collections.abc import Mapping, Sequence +from collections.abc import Generator, Mapping, Sequence from dataclasses import dataclass from typing import Any @@ -32,7 +32,7 @@ class _FitAndScoreJob: @dataclass(frozen=True) -class _GSResult: +class GSResult: r"""Result from a single grid search evaluation.""" metalearner: MetaLearner @@ -42,7 +42,7 @@ class _GSResult: score_time: float -def _fit_and_score(job: _FitAndScoreJob) -> _GSResult: +def _fit_and_score(job: _FitAndScoreJob) -> GSResult: start_time = time.time() job.metalearner.fit( job.X_train, job.y_train, job.w_train, **job.metalerner_fit_params @@ -69,7 +69,7 @@ def _fit_and_score(job: _FitAndScoreJob) -> _GSResult: else: test_scores = None score_time = time.time() - start_time - return _GSResult( + return GSResult( metalearner=job.metalearner, fit_time=fit_time, score_time=score_time, @@ -78,7 +78,9 @@ def _fit_and_score(job: _FitAndScoreJob) -> _GSResult: ) -def _format_results(results: Sequence[_GSResult]) -> pd.DataFrame: +def _format_results( + results: list[GSResult] | Generator[GSResult, None, None] +) -> pd.DataFrame: rows = [] for result in results: row: dict[str, str | int | float] = {} @@ -180,7 +182,23 @@ class MetaLearnerGridSearch: ``verbose`` will be passed to `joblib.Parallel `_. - After fitting a dataframe with the results will be available in `results_`. + ``store_raw_results`` and ``store_results`` define which and how the results are saved + after calling :meth:`~metalearners.grid_search.MetaLearnerGridSearch.fit` depending on + their values: + + * Both are ``True`` (default): ``raw_results_`` will be a list of + :class:`~metalearners.grid_search.GSResult` with all the results and ``results_`` + will be a DataFrame with the processed results. + * ``store_raw_results=True`` and ``store_results=False``: ``raw_results_`` will be a + list of :class:`~metalearners.grid_search.GSResult` with all the results + and ``results`` will be ``None``. + * ``store_raw_results=False`` and ``store_results=True``: ``raw_results_`` will be + ``None`` and ``results_`` will be a DataFrame with the processed results. + * Both are ``False``: ``raw_results_`` will be a generator which yields a + :class:`~metalearners.grid_search.GSResult` for each configuration and ``results`` + will be None. This configuration can be useful in the case the grid search is big + and you do not want to store all MetaLearners objects rather evaluate them after + fitting each one and just store one. """ # TODO: Add a reference to a docs example once it is written. @@ -195,6 +213,8 @@ def __init__( n_jobs: int | None = None, random_state: int | None = None, verbose: int = 0, + store_raw_results: bool = True, + store_results: bool = True, ): self.metalearner_factory = metalearner_factory self.metalearner_params = metalearner_params @@ -202,8 +222,12 @@ def __init__( self.n_jobs = n_jobs self.random_state = random_state self.verbose = verbose + self.store_raw_results = store_raw_results + self.store_results = store_results - self.raw_results_: Sequence[_GSResult] | None = None + self.raw_results_: list[GSResult] | Generator[GSResult, None, None] | None = ( + None + ) self.results_: pd.DataFrame | None = None all_base_models = set( @@ -311,8 +335,19 @@ def fit( metalerner_fit_params=kwargs, ) ) - - parallel = Parallel(n_jobs=self.n_jobs, verbose=self.verbose) + return_as = ( + "list" if self.store_raw_results else "generator" + ) # Can we use generator_unordered? + parallel = Parallel( + n_jobs=self.n_jobs, verbose=self.verbose, return_as=return_as + ) raw_results = parallel(delayed(_fit_and_score)(job) for job in jobs) self.raw_results_ = raw_results - self.results_ = _format_results(results=raw_results) + if self.store_results: + self.results_ = _format_results(results=raw_results) + if not self.store_raw_results: + # This just checks that the generator is empty + try: + next(self.raw_results_) # type: ignore + except StopIteration: + self.raw_results_ = None From e8e3b395ac1eb1c39c000803cfe91ace0d02be30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Wed, 17 Jul 2024 13:52:03 +0200 Subject: [PATCH 02/14] Tests --- tests/test_grid_search.py | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/test_grid_search.py b/tests/test_grid_search.py index e29d3d3..5949b86 100644 --- a/tests/test_grid_search.py +++ b/tests/test_grid_search.py @@ -2,7 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause +from types import GeneratorType + import numpy as np +import pandas as pd import pytest from lightgbm import LGBMClassifier, LGBMRegressor from sklearn.linear_model import LinearRegression, LogisticRegression @@ -259,3 +262,42 @@ def test_metalearnergridsearch_reuse_propensity_smoke(grid_search_data): assert gs.results_ is not None assert gs.results_.shape[0] == 2 assert len(gs.results_.index.names) == 5 + + +@pytest.mark.parametrize( + "store_raw_results, store_results, expected_type_raw_results, expected_type_results", + [ + (True, True, list, pd.DataFrame), + (True, False, list, type(None)), + (False, True, type(None), pd.DataFrame), + (False, False, GeneratorType, type(None)), + ], +) +def test_metalearnergridsearch_store( + store_raw_results, + store_results, + expected_type_raw_results, + expected_type_results, + grid_search_data, +): + X, _, y, w, X_test, _, y_test, w_test = grid_search_data + n_variants = len(np.unique(w)) + + metalearner_params = { + "is_classification": False, + "n_variants": n_variants, + "n_folds": 2, + } + + gs = MetaLearnerGridSearch( + metalearner_factory=SLearner, + metalearner_params=metalearner_params, + base_learner_grid={"base_model": [LinearRegression, LGBMRegressor]}, + param_grid={"base_model": {"LGBMRegressor": {"n_estimators": [1, 2]}}}, + store_raw_results=store_raw_results, + store_results=store_results, + ) + + gs.fit(X, y, w, X_test, y_test, w_test) + assert isinstance(gs.raw_results_, expected_type_raw_results) + assert isinstance(gs.results_, expected_type_results) From 44fa6ec0d918027f71353d52a1befe4e054232e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Wed, 17 Jul 2024 13:55:18 +0200 Subject: [PATCH 03/14] Finish TODO --- metalearners/grid_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index fdd8609..9879092 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -199,9 +199,9 @@ class MetaLearnerGridSearch: will be None. This configuration can be useful in the case the grid search is big and you do not want to store all MetaLearners objects rather evaluate them after fitting each one and just store one. - """ - # TODO: Add a reference to a docs example once it is written. + For an illustration see :ref:`our example on Tuning hyperparameters of a MetaLearner with MetaLearnerGridSearch `. + """ def __init__( self, From 629999dbbfd2f846a3e92b9b56be63ad784b906e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Wed, 17 Jul 2024 15:40:09 +0200 Subject: [PATCH 04/14] Reduce memory usage by not creating metalearner object --- metalearners/grid_search.py | 49 +++++++++++++++++++++++-------------- tests/test_grid_search.py | 23 +++++++++++++++++ 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index 9879092..505cb4e 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -17,7 +17,8 @@ @dataclass(frozen=True) class _FitAndScoreJob: - metalearner: MetaLearner + metalearner_factory: type[MetaLearner] + metalearner_params: dict[str, Any] X_train: Matrix y_train: Vector w_train: Vector @@ -44,13 +45,12 @@ class GSResult: def _fit_and_score(job: _FitAndScoreJob) -> GSResult: start_time = time.time() - job.metalearner.fit( - job.X_train, job.y_train, job.w_train, **job.metalerner_fit_params - ) + ml = job.metalearner_factory(**job.metalearner_params) + ml.fit(job.X_train, job.y_train, job.w_train, **job.metalerner_fit_params) fit_time = time.time() - start_time start_time = time.time() - train_scores = job.metalearner.evaluate( + train_scores = ml.evaluate( X=job.X_train, y=job.y_train, w=job.w_train, @@ -58,7 +58,7 @@ def _fit_and_score(job: _FitAndScoreJob) -> GSResult: scoring=job.scoring, ) if job.X_test is not None and job.y_test is not None and job.w_test is not None: - test_scores = job.metalearner.evaluate( + test_scores = ml.evaluate( X=job.X_test, y=job.y_test, w=job.w_test, @@ -70,7 +70,7 @@ def _fit_and_score(job: _FitAndScoreJob) -> GSResult: test_scores = None score_time = time.time() - start_time return GSResult( - metalearner=job.metalearner, + metalearner=ml, fit_time=fit_time, score_time=score_time, train_scores=train_scores, @@ -310,20 +310,33 @@ def fit( } propensity_model_params = params.get(PROPENSITY_MODEL, None) - ml = self.metalearner_factory( - **self.metalearner_params, - nuisance_model_factory=nuisance_model_factory, - treatment_model_factory=treatment_model_factory, - propensity_model_factory=propensity_model_factory, - nuisance_model_params=nuisance_model_params, - treatment_model_params=treatment_model_params, - propensity_model_params=propensity_model_params, - random_state=self.random_state, - ) + grid_metalearner_params = { + "nuisance_model_factory": nuisance_model_factory, + "treatment_model_factory": treatment_model_factory, + "propensity_model_factory": propensity_model_factory, + "nuisance_model_params": nuisance_model_params, + "treatment_model_params": treatment_model_params, + "propensity_model_params": propensity_model_params, + "random_state": self.random_state, + } + + if ( + len( + shared_keys := set(grid_metalearner_params.keys()) + & set(self.metalearner_params.keys()) + ) + > 0 + ): + raise ValueError( + f"{shared_keys} should not be specified in metalearner_params as " + "they are used internally. Please use the correct parameters." + ) jobs.append( _FitAndScoreJob( - metalearner=ml, + metalearner_factory=self.metalearner_factory, + metalearner_params=dict(self.metalearner_params) + | grid_metalearner_params, X_train=X, y_train=y, w_train=w, diff --git a/tests/test_grid_search.py b/tests/test_grid_search.py index 5949b86..5d5e4d8 100644 --- a/tests/test_grid_search.py +++ b/tests/test_grid_search.py @@ -301,3 +301,26 @@ def test_metalearnergridsearch_store( gs.fit(X, y, w, X_test, y_test, w_test) assert isinstance(gs.raw_results_, expected_type_raw_results) assert isinstance(gs.results_, expected_type_results) + + +def test_metalearnergridsearch_error(grid_search_data): + X, _, y, w, X_test, _, y_test, w_test = grid_search_data + n_variants = len(np.unique(w)) + + metalearner_params = { + "is_classification": False, + "n_variants": n_variants, + "n_folds": 2, + "random_state": 1, + } + + gs = MetaLearnerGridSearch( + metalearner_factory=SLearner, + metalearner_params=metalearner_params, + base_learner_grid={"base_model": [LinearRegression, LGBMRegressor]}, + param_grid={"base_model": {"LGBMRegressor": {"n_estimators": [1, 2]}}}, + ) + with pytest.raises( + ValueError, match="should not be specified in metalearner_params" + ): + gs.fit(X, y, w, X_test, y_test, w_test) From 6da41803bf38ed9b05cab631646c047cbbb5608c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Wed, 17 Jul 2024 15:46:15 +0200 Subject: [PATCH 05/14] Update CHANGELOG --- CHANGELOG.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d04fad1..9095266 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,16 @@ Changelog ========= +0.8.0 (2024-07-xx) +------------------ + +**New features** + +* Add optional ``store_raw_results`` and ``store_results`` parameters to :class:`metalearners.grid_search.MetaLearnerGridSearch`. + +* Renamed :class:`metalearners.grid_search._GSResult` to :class:`metalearners.grid_search.GSResult`. + + 0.7.0 (2024-07-12) ------------------ From 1e945ba765c4750cb4247401dc1ab49fdb6dc452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Wed, 17 Jul 2024 15:48:18 +0200 Subject: [PATCH 06/14] Use generator_unordered --- metalearners/grid_search.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index 505cb4e..7dfa2ea 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -348,9 +348,7 @@ def fit( metalerner_fit_params=kwargs, ) ) - return_as = ( - "list" if self.store_raw_results else "generator" - ) # Can we use generator_unordered? + return_as = "list" if self.store_raw_results else "generator_unordered" parallel = Parallel( n_jobs=self.n_jobs, verbose=self.verbose, return_as=return_as ) From 4682e5372e5cf9b490e20e8c732416777d8e6185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Wed, 17 Jul 2024 16:37:20 +0200 Subject: [PATCH 07/14] Add grid_size_ and move attributes initialization to fit --- CHANGELOG.rst | 2 ++ metalearners/grid_search.py | 15 +++++++-------- tests/test_grid_search.py | 1 + 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9095266..13a0dab 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,8 @@ Changelog * Renamed :class:`metalearners.grid_search._GSResult` to :class:`metalearners.grid_search.GSResult`. +* Added ``grid_size_`` attribute to :class:`metalearners.grid_search.MetaLearnerGridSearch`. + 0.7.0 (2024-07-12) ------------------ diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index 7dfa2ea..304beb5 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -225,11 +225,6 @@ def __init__( self.store_raw_results = store_raw_results self.store_results = store_results - self.raw_results_: list[GSResult] | Generator[GSResult, None, None] | None = ( - None - ) - self.results_: pd.DataFrame | None = None - all_base_models = set( metalearner_factory.nuisance_model_specifications().keys() ) | set(metalearner_factory.treatment_model_specifications().keys()) @@ -348,14 +343,18 @@ def fit( metalerner_fit_params=kwargs, ) ) + + self.grid_size_ = len(jobs) + self.raw_results_: list[GSResult] | Generator[GSResult, None, None] | None + self.results_: pd.DataFrame | None + return_as = "list" if self.store_raw_results else "generator_unordered" parallel = Parallel( n_jobs=self.n_jobs, verbose=self.verbose, return_as=return_as ) - raw_results = parallel(delayed(_fit_and_score)(job) for job in jobs) - self.raw_results_ = raw_results + self.raw_results = parallel(delayed(_fit_and_score)(job) for job in jobs) if self.store_results: - self.results_ = _format_results(results=raw_results) + self.results_ = _format_results(results=self.raw_results) if not self.store_raw_results: # This just checks that the generator is empty try: diff --git a/tests/test_grid_search.py b/tests/test_grid_search.py index 5d5e4d8..23e4cbd 100644 --- a/tests/test_grid_search.py +++ b/tests/test_grid_search.py @@ -156,6 +156,7 @@ def test_metalearnergridsearch_smoke( assert gs.results_ is not None assert gs.results_.shape[0] == expected_n_configs assert gs.results_.index.names == expected_index_cols + assert gs.grid_size_ == expected_n_configs train_scores_cols = set( c[6:] for c in list(gs.results_.columns) if c.startswith("train_") From b1fd5b8ad9f2d9970f803866269e89ddd205a417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Wed, 17 Jul 2024 17:11:31 +0200 Subject: [PATCH 08/14] Fix --- metalearners/grid_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index 304beb5..16b120d 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -352,9 +352,9 @@ def fit( parallel = Parallel( n_jobs=self.n_jobs, verbose=self.verbose, return_as=return_as ) - self.raw_results = parallel(delayed(_fit_and_score)(job) for job in jobs) + self.raw_results_ = parallel(delayed(_fit_and_score)(job) for job in jobs) if self.store_results: - self.results_ = _format_results(results=self.raw_results) + self.results_ = _format_results(results=self.raw_results_) # type: ignore if not self.store_raw_results: # This just checks that the generator is empty try: From 4ef8e6eb90dd09df28242a58b39a5281cdc21a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Wed, 17 Jul 2024 17:57:29 +0200 Subject: [PATCH 09/14] Fix --- metalearners/grid_search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index 16b120d..020ae9c 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -361,3 +361,5 @@ def fit( next(self.raw_results_) # type: ignore except StopIteration: self.raw_results_ = None + else: + self.results_ = None From 950dda3459594031c326cc5854c6919b97540bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Thu, 18 Jul 2024 09:23:54 +0200 Subject: [PATCH 10/14] grid_size_ docstring --- metalearners/grid_search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index 020ae9c..651c0c5 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -200,6 +200,8 @@ class MetaLearnerGridSearch: and you do not want to store all MetaLearners objects rather evaluate them after fitting each one and just store one. + ``grid_size_`` will contain the number of hyperparameter combinations after fitting. + For an illustration see :ref:`our example on Tuning hyperparameters of a MetaLearner with MetaLearnerGridSearch `. """ From 46a88ccc74605bfe4a981a0ff7b57c04f4ae5bb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Fri, 19 Jul 2024 09:11:01 +0200 Subject: [PATCH 11/14] Add new options to tutorial --- docs/examples/example_gridsearch.ipynb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/examples/example_gridsearch.ipynb b/docs/examples/example_gridsearch.ipynb index 586c5d0..ada0a00 100644 --- a/docs/examples/example_gridsearch.ipynb +++ b/docs/examples/example_gridsearch.ipynb @@ -327,6 +327,26 @@ "gs.results_" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What if I run out of memory?\n", + "----------------------------\n", + "\n", + "If you're conducting an optimization task over a large grid with a substantial dataset,\n", + "it is possible that memory usage issues may arise. To try to solve these, you can minimize\n", + "memory usage by adjusting your settings.\n", + "\n", + "In that case you can set ``store_raw_results=False``, the grid search will then operate\n", + "with a generator rather than a list, significantly reducing memory usage.\n", + "\n", + "If the ``results_ DataFrame`` is what you're after, you can simply set ``store_results=True``.\n", + "However, if you aim to iterate over the {class}`~metalearners.metalearner.MetaLearner` objects,\n", + "you can set ``store_results=False``. Consequently, ``raw_results_`` will become a generator\n", + "object yielding {class}`~metalearners.grid_search.GSResult`." + ] + }, { "cell_type": "markdown", "metadata": {}, From e34b7512c9c8f2e0799d3e77522c257a4a82fabb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Mon, 22 Jul 2024 09:17:22 +0200 Subject: [PATCH 12/14] Remove check empty generator --- metalearners/grid_search.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index 651c0c5..67c574b 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -358,10 +358,7 @@ def fit( if self.store_results: self.results_ = _format_results(results=self.raw_results_) # type: ignore if not self.store_raw_results: - # This just checks that the generator is empty - try: - next(self.raw_results_) # type: ignore - except StopIteration: - self.raw_results_ = None + # The generator will be empty so we replace it with None + self.raw_results_ = None else: self.results_ = None From 4c1c12d50ad983419b115b6616ab5ec977ed37e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= <154450563+FrancescMartiEscofetQC@users.noreply.github.com> Date: Mon, 22 Jul 2024 11:52:04 +0200 Subject: [PATCH 13/14] Apply suggestions from code review Co-authored-by: Kevin Klein <7267523+kklein@users.noreply.github.com> --- metalearners/grid_search.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index 67c574b..c5ed4f2 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -348,7 +348,7 @@ def fit( self.grid_size_ = len(jobs) self.raw_results_: list[GSResult] | Generator[GSResult, None, None] | None - self.results_: pd.DataFrame | None + self.results_: pd.DataFrame | None = None return_as = "list" if self.store_raw_results else "generator_unordered" parallel = Parallel( @@ -360,5 +360,3 @@ def fit( if not self.store_raw_results: # The generator will be empty so we replace it with None self.raw_results_ = None - else: - self.results_ = None From 1590e9456c4985de4ed61a9a54dd86c96576eed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Mart=C3=AD=20Escofet?= Date: Mon, 22 Jul 2024 17:21:14 +0200 Subject: [PATCH 14/14] Add explanation grid_size_ --- metalearners/grid_search.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metalearners/grid_search.py b/metalearners/grid_search.py index c5ed4f2..93f1981 100644 --- a/metalearners/grid_search.py +++ b/metalearners/grid_search.py @@ -201,6 +201,10 @@ class MetaLearnerGridSearch: fitting each one and just store one. ``grid_size_`` will contain the number of hyperparameter combinations after fitting. + This attribute may be useful in the case ``store_raw_results = False`` and ``store_results = False``. + In that case, the generator object returned in ``raw_results_`` doesn't trigger the fitting + of individual metalearners until explicitly requested, e.g. in a loop. This attribute + can be use to track the progress, for instance, by creating a progress bar or a similar utility. For an illustration see :ref:`our example on Tuning hyperparameters of a MetaLearner with MetaLearnerGridSearch `. """