From 8d3da4a58a7f837d51ea9e50a22548e9a25c3fd8 Mon Sep 17 00:00:00 2001 From: dheemantha Date: Fri, 22 Dec 2023 17:11:05 +0100 Subject: [PATCH 1/6] Fetched from https://github.com/optuna/optuna/commit/dd3b87e4a36a25ca08a85a586767aa9c7a268c2c --- optuna/integration/tensorboard.py | 121 ++++++++++++++++++++++++++++++ optuna_integration/tensorboard.py | 121 ++++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 optuna/integration/tensorboard.py create mode 100644 optuna_integration/tensorboard.py diff --git a/optuna/integration/tensorboard.py b/optuna/integration/tensorboard.py new file mode 100644 index 000000000..9f588c650 --- /dev/null +++ b/optuna/integration/tensorboard.py @@ -0,0 +1,121 @@ +import os +from typing import Dict + +import optuna +from optuna._experimental import experimental_class +from optuna._imports import try_import +from optuna.logging import get_logger + + +with try_import() as _imports: + from tensorboard.plugins.hparams import api as hp + import tensorflow as tf + +_logger = get_logger(__name__) + + +@experimental_class("2.0.0") +class TensorBoardCallback: + """Callback to track Optuna trials with TensorBoard. + + This callback adds relevant information that is tracked by Optuna to TensorBoard. + + See `the example `_. + + Args: + dirname: + Directory to store TensorBoard logs. + metric_name: + Name of the metric. Since the metric itself is just a number, + `metric_name` can be used to give it a name. So you know later + if it was roc-auc or accuracy. + + """ + + def __init__(self, dirname: str, metric_name: str) -> None: + _imports.check() + self._dirname = dirname + self._metric_name = metric_name + self._hp_params: Dict[str, hp.HParam] = {} + + def __call__(self, study: optuna.study.Study, trial: optuna.trial.FrozenTrial) -> None: + if len(self._hp_params) == 0: + self._initialization(study) + if trial.state != optuna.trial.TrialState.COMPLETE: + return + trial_value = trial.value if trial.value is not None else float("nan") + hparams = {} + for param_name, param_value in trial.params.items(): + if param_name not in self._hp_params: + self._add_distributions(trial.distributions) + param = self._hp_params[param_name] + if isinstance(param.domain, hp.Discrete): + hparams[param] = param.domain.dtype(param_value) + else: + hparams[param] = param_value + run_name = "trial-%d" % trial.number + run_dir = os.path.join(self._dirname, run_name) + with tf.summary.create_file_writer(run_dir).as_default(): + hp.hparams(hparams, trial_id=run_name) # record the values used in this trial + tf.summary.scalar(self._metric_name, trial_value, step=trial.number) + + def _add_distributions( + self, distributions: Dict[str, optuna.distributions.BaseDistribution] + ) -> None: + supported_distributions = ( + optuna.distributions.CategoricalDistribution, + optuna.distributions.FloatDistribution, + optuna.distributions.IntDistribution, + ) + + for param_name, param_distribution in distributions.items(): + if isinstance(param_distribution, optuna.distributions.FloatDistribution): + self._hp_params[param_name] = hp.HParam( + param_name, + hp.RealInterval(float(param_distribution.low), float(param_distribution.high)), + ) + elif isinstance(param_distribution, optuna.distributions.IntDistribution): + self._hp_params[param_name] = hp.HParam( + param_name, + hp.IntInterval(param_distribution.low, param_distribution.high), + ) + elif isinstance(param_distribution, optuna.distributions.CategoricalDistribution): + choices = param_distribution.choices + dtype = type(choices[0]) + if any(not isinstance(choice, dtype) for choice in choices): + _logger.warning( + "Choices contains mixed types, which is not supported by TensorBoard. " + "Converting all choices to strings." + ) + choices = tuple(map(str, choices)) + elif dtype not in (int, float, bool, str): + _logger.warning( + f"Choices are of type {dtype}, which is not supported by TensorBoard. " + "Converting all choices to strings." + ) + choices = tuple(map(str, choices)) + + self._hp_params[param_name] = hp.HParam( + param_name, + hp.Discrete(choices), + ) + else: + distribution_list = [ + distribution.__name__ for distribution in supported_distributions + ] + raise NotImplementedError( + "The distribution {} is not implemented. " + "The parameter distribution should be one of the {}".format( + param_distribution, distribution_list + ) + ) + + def _initialization(self, study: optuna.Study) -> None: + completed_trials = [ + trial + for trial in study.get_trials(deepcopy=False) + if trial.state == optuna.trial.TrialState.COMPLETE + ] + for trial in completed_trials: + self._add_distributions(trial.distributions) diff --git a/optuna_integration/tensorboard.py b/optuna_integration/tensorboard.py new file mode 100644 index 000000000..9f588c650 --- /dev/null +++ b/optuna_integration/tensorboard.py @@ -0,0 +1,121 @@ +import os +from typing import Dict + +import optuna +from optuna._experimental import experimental_class +from optuna._imports import try_import +from optuna.logging import get_logger + + +with try_import() as _imports: + from tensorboard.plugins.hparams import api as hp + import tensorflow as tf + +_logger = get_logger(__name__) + + +@experimental_class("2.0.0") +class TensorBoardCallback: + """Callback to track Optuna trials with TensorBoard. + + This callback adds relevant information that is tracked by Optuna to TensorBoard. + + See `the example `_. + + Args: + dirname: + Directory to store TensorBoard logs. + metric_name: + Name of the metric. Since the metric itself is just a number, + `metric_name` can be used to give it a name. So you know later + if it was roc-auc or accuracy. + + """ + + def __init__(self, dirname: str, metric_name: str) -> None: + _imports.check() + self._dirname = dirname + self._metric_name = metric_name + self._hp_params: Dict[str, hp.HParam] = {} + + def __call__(self, study: optuna.study.Study, trial: optuna.trial.FrozenTrial) -> None: + if len(self._hp_params) == 0: + self._initialization(study) + if trial.state != optuna.trial.TrialState.COMPLETE: + return + trial_value = trial.value if trial.value is not None else float("nan") + hparams = {} + for param_name, param_value in trial.params.items(): + if param_name not in self._hp_params: + self._add_distributions(trial.distributions) + param = self._hp_params[param_name] + if isinstance(param.domain, hp.Discrete): + hparams[param] = param.domain.dtype(param_value) + else: + hparams[param] = param_value + run_name = "trial-%d" % trial.number + run_dir = os.path.join(self._dirname, run_name) + with tf.summary.create_file_writer(run_dir).as_default(): + hp.hparams(hparams, trial_id=run_name) # record the values used in this trial + tf.summary.scalar(self._metric_name, trial_value, step=trial.number) + + def _add_distributions( + self, distributions: Dict[str, optuna.distributions.BaseDistribution] + ) -> None: + supported_distributions = ( + optuna.distributions.CategoricalDistribution, + optuna.distributions.FloatDistribution, + optuna.distributions.IntDistribution, + ) + + for param_name, param_distribution in distributions.items(): + if isinstance(param_distribution, optuna.distributions.FloatDistribution): + self._hp_params[param_name] = hp.HParam( + param_name, + hp.RealInterval(float(param_distribution.low), float(param_distribution.high)), + ) + elif isinstance(param_distribution, optuna.distributions.IntDistribution): + self._hp_params[param_name] = hp.HParam( + param_name, + hp.IntInterval(param_distribution.low, param_distribution.high), + ) + elif isinstance(param_distribution, optuna.distributions.CategoricalDistribution): + choices = param_distribution.choices + dtype = type(choices[0]) + if any(not isinstance(choice, dtype) for choice in choices): + _logger.warning( + "Choices contains mixed types, which is not supported by TensorBoard. " + "Converting all choices to strings." + ) + choices = tuple(map(str, choices)) + elif dtype not in (int, float, bool, str): + _logger.warning( + f"Choices are of type {dtype}, which is not supported by TensorBoard. " + "Converting all choices to strings." + ) + choices = tuple(map(str, choices)) + + self._hp_params[param_name] = hp.HParam( + param_name, + hp.Discrete(choices), + ) + else: + distribution_list = [ + distribution.__name__ for distribution in supported_distributions + ] + raise NotImplementedError( + "The distribution {} is not implemented. " + "The parameter distribution should be one of the {}".format( + param_distribution, distribution_list + ) + ) + + def _initialization(self, study: optuna.Study) -> None: + completed_trials = [ + trial + for trial in study.get_trials(deepcopy=False) + if trial.state == optuna.trial.TrialState.COMPLETE + ] + for trial in completed_trials: + self._add_distributions(trial.distributions) From c7b3155f30da3d44af5be27906ecaf3267d3960f Mon Sep 17 00:00:00 2001 From: dheemantha Date: Fri, 22 Dec 2023 17:12:15 +0100 Subject: [PATCH 2/6] Fetched from https://github.com/optuna/optuna/commit/b94671d1c5fe2606d1d9c085cc346dee2ddb17a0 --- tests/test_tensorboard.py | 113 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tests/test_tensorboard.py diff --git a/tests/test_tensorboard.py b/tests/test_tensorboard.py new file mode 100644 index 000000000..7817c6f6d --- /dev/null +++ b/tests/test_tensorboard.py @@ -0,0 +1,113 @@ +import os +import shutil +import tempfile + +import pytest + +import optuna +from optuna._imports import try_import +from optuna.integration.tensorboard import TensorBoardCallback + + +with try_import(): + from tensorboard.backend.event_processing.event_accumulator import EventAccumulator + +pytestmark = pytest.mark.integration + + +def _objective_func(trial: optuna.trial.Trial) -> float: + u = trial.suggest_int("u", 0, 10, step=2) + v = trial.suggest_int("v", 1, 10, log=True) + w = trial.suggest_float("w", -1.0, 1.0, step=0.1) + x = trial.suggest_float("x", -1.0, 1.0) + y = trial.suggest_float("y", 20.0, 30.0, log=True) + z = trial.suggest_categorical("z", (-1.0, 1.0)) + trial.set_user_attr("my_user_attr", "my_user_attr_value") + return u + v + w + (x - 2) ** 2 + (y - 25) ** 2 + z + + +def test_study_name() -> None: + dirname = tempfile.mkdtemp() + metric_name = "target" + study_name = "test_tensorboard_integration" + + tbcallback = TensorBoardCallback(dirname, metric_name) + study = optuna.create_study(study_name=study_name) + study.optimize(_objective_func, n_trials=1, callbacks=[tbcallback]) + + event_acc = EventAccumulator(os.path.join(dirname, "trial-0")) + event_acc.Reload() + + try: + assert len(event_acc.Tensors("target")) == 1 + except Exception as e: + raise e + finally: + shutil.rmtree(dirname) + + +def test_cast_float() -> None: + def objective(trial: optuna.trial.Trial) -> float: + x = trial.suggest_float("x", 1, 2) + y = trial.suggest_float("y", 1, 2, log=True) + assert isinstance(x, float) + assert isinstance(y, float) + return x + y + + dirname = tempfile.mkdtemp() + metric_name = "target" + study_name = "test_tensorboard_integration" + + tbcallback = TensorBoardCallback(dirname, metric_name) + study = optuna.create_study(study_name=study_name) + study.optimize(objective, n_trials=1, callbacks=[tbcallback]) + + +def test_categorical() -> None: + def objective(trial: optuna.trial.Trial) -> float: + x = trial.suggest_categorical("x", [1, 2, 3]) + assert isinstance(x, int) + return x + + dirname = tempfile.mkdtemp() + metric_name = "target" + study_name = "test_tensorboard_integration" + + tbcallback = TensorBoardCallback(dirname, metric_name) + study = optuna.create_study(study_name=study_name) + study.optimize(objective, n_trials=1, callbacks=[tbcallback]) + + +def test_categorical_mixed_types() -> None: + def objective(trial: optuna.trial.Trial) -> float: + x = trial.suggest_categorical("x", [None, 1, 2, 3.14, True, "foo"]) + assert x is None or isinstance(x, (int, float, bool, str)) + return len(str(x)) + + dirname = tempfile.mkdtemp() + metric_name = "target" + study_name = "test_tensorboard_integration" + + tbcallback = TensorBoardCallback(dirname, metric_name) + study = optuna.create_study(study_name=study_name) + study.optimize(objective, n_trials=10, callbacks=[tbcallback]) + + +def test_categorical_unsupported_types() -> None: + def objective(trial: optuna.trial.Trial) -> float: + x = trial.suggest_categorical("x", [[1, 2], [3, 4, 5], [6]]) # type: ignore[list-item] + assert isinstance(x, list) + return len(x) + + dirname = tempfile.mkdtemp() + metric_name = "target" + study_name = "test_tensorboard_integration" + + tbcallback = TensorBoardCallback(dirname, metric_name) + study = optuna.create_study(study_name=study_name) + study.optimize(objective, n_trials=10, callbacks=[tbcallback]) + + +def test_experimental_warning() -> None: + with pytest.warns(optuna.exceptions.ExperimentalWarning): + TensorBoardCallback(dirname="", metric_name="") From 80c8865e2d06ea91ec57fadc7db81331ba9c2c19 Mon Sep 17 00:00:00 2001 From: dheemantha Date: Fri, 22 Dec 2023 17:14:33 +0100 Subject: [PATCH 3/6] Tensorboard integration --- README.md | 1 + docs/source/reference/index.rst | 12 ++- optuna/integration/tensorboard.py | 121 ------------------------------ pyproject.toml | 1 + 4 files changed, 13 insertions(+), 122 deletions(-) delete mode 100644 optuna/integration/tensorboard.py diff --git a/README.md b/README.md index f5bab8c3e..11e9d6e43 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Optuna-Integration API reference is [here](https://optuna-integration.readthedoc * [MXNet](https://optuna-integration.readthedocs.io/en/stable/reference/index.html#mxnet) ([example](https://github.com/optuna/optuna-examples/tree/main/mxnet)) * [SHAP](https://optuna-integration.readthedocs.io/en/stable/reference/index.html#shap) * [skorch](https://optuna-integration.readthedocs.io/en/stable/reference/index.html#skorch) ([example](https://github.com/optuna/optuna-examples/tree/main/pytorch/skorch_simple.py)) +* [TensorBoard](https://optuna-integration.readthedocs.io/en/stable/reference/index.html#tensorboard) ([example](https://github.com/optuna/optuna-examples/tree/main/tensorboard/tensorboard_simple.py)) * [tf.keras](https://optuna-integration.readthedocs.io/en/stable/reference/index.html#tensorflow) ([example](https://github.com/optuna/optuna-examples/tree/main/tfkeras/tfkeras_integration.py)) ## Installation diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index cfbb740ab..26cd17c6d 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -93,6 +93,15 @@ skorch optuna.integration.SkorchPruningCallback +TensorBoard +---------- + +.. autosummary:: + :toctree: generated/ + :nosignatures: + + optuna.integration.TensorBoardCallback + TensorFlow ---------- @@ -100,4 +109,5 @@ TensorFlow :toctree: generated/ :nosignatures: - optuna.integration.TFKerasPruningCallback \ No newline at end of file + optuna.integration.TFKerasPruningCallback + \ No newline at end of file diff --git a/optuna/integration/tensorboard.py b/optuna/integration/tensorboard.py deleted file mode 100644 index 9f588c650..000000000 --- a/optuna/integration/tensorboard.py +++ /dev/null @@ -1,121 +0,0 @@ -import os -from typing import Dict - -import optuna -from optuna._experimental import experimental_class -from optuna._imports import try_import -from optuna.logging import get_logger - - -with try_import() as _imports: - from tensorboard.plugins.hparams import api as hp - import tensorflow as tf - -_logger = get_logger(__name__) - - -@experimental_class("2.0.0") -class TensorBoardCallback: - """Callback to track Optuna trials with TensorBoard. - - This callback adds relevant information that is tracked by Optuna to TensorBoard. - - See `the example `_. - - Args: - dirname: - Directory to store TensorBoard logs. - metric_name: - Name of the metric. Since the metric itself is just a number, - `metric_name` can be used to give it a name. So you know later - if it was roc-auc or accuracy. - - """ - - def __init__(self, dirname: str, metric_name: str) -> None: - _imports.check() - self._dirname = dirname - self._metric_name = metric_name - self._hp_params: Dict[str, hp.HParam] = {} - - def __call__(self, study: optuna.study.Study, trial: optuna.trial.FrozenTrial) -> None: - if len(self._hp_params) == 0: - self._initialization(study) - if trial.state != optuna.trial.TrialState.COMPLETE: - return - trial_value = trial.value if trial.value is not None else float("nan") - hparams = {} - for param_name, param_value in trial.params.items(): - if param_name not in self._hp_params: - self._add_distributions(trial.distributions) - param = self._hp_params[param_name] - if isinstance(param.domain, hp.Discrete): - hparams[param] = param.domain.dtype(param_value) - else: - hparams[param] = param_value - run_name = "trial-%d" % trial.number - run_dir = os.path.join(self._dirname, run_name) - with tf.summary.create_file_writer(run_dir).as_default(): - hp.hparams(hparams, trial_id=run_name) # record the values used in this trial - tf.summary.scalar(self._metric_name, trial_value, step=trial.number) - - def _add_distributions( - self, distributions: Dict[str, optuna.distributions.BaseDistribution] - ) -> None: - supported_distributions = ( - optuna.distributions.CategoricalDistribution, - optuna.distributions.FloatDistribution, - optuna.distributions.IntDistribution, - ) - - for param_name, param_distribution in distributions.items(): - if isinstance(param_distribution, optuna.distributions.FloatDistribution): - self._hp_params[param_name] = hp.HParam( - param_name, - hp.RealInterval(float(param_distribution.low), float(param_distribution.high)), - ) - elif isinstance(param_distribution, optuna.distributions.IntDistribution): - self._hp_params[param_name] = hp.HParam( - param_name, - hp.IntInterval(param_distribution.low, param_distribution.high), - ) - elif isinstance(param_distribution, optuna.distributions.CategoricalDistribution): - choices = param_distribution.choices - dtype = type(choices[0]) - if any(not isinstance(choice, dtype) for choice in choices): - _logger.warning( - "Choices contains mixed types, which is not supported by TensorBoard. " - "Converting all choices to strings." - ) - choices = tuple(map(str, choices)) - elif dtype not in (int, float, bool, str): - _logger.warning( - f"Choices are of type {dtype}, which is not supported by TensorBoard. " - "Converting all choices to strings." - ) - choices = tuple(map(str, choices)) - - self._hp_params[param_name] = hp.HParam( - param_name, - hp.Discrete(choices), - ) - else: - distribution_list = [ - distribution.__name__ for distribution in supported_distributions - ] - raise NotImplementedError( - "The distribution {} is not implemented. " - "The parameter distribution should be one of the {}".format( - param_distribution, distribution_list - ) - ) - - def _initialization(self, study: optuna.Study) -> None: - completed_trials = [ - trial - for trial in study.get_trials(deepcopy=False) - if trial.state == optuna.trial.TrialState.COMPLETE - ] - for trial in completed_trials: - self._add_distributions(trial.distributions) diff --git a/pyproject.toml b/pyproject.toml index 2e082a121..3e340a2fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ all = [ "mxnet", "shap", "skorch", + "tensorboard", "tensorflow", ] From b4bf450dd1a2e33440b9585a998f97c66c63b76c Mon Sep 17 00:00:00 2001 From: dheemantha Date: Fri, 22 Dec 2023 17:45:22 +0100 Subject: [PATCH 4/6] updated imports --- optuna_integration/tensorboard.py | 2 +- tests/test_tensorboard.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/optuna_integration/tensorboard.py b/optuna_integration/tensorboard.py index 9f588c650..e79479eab 100644 --- a/optuna_integration/tensorboard.py +++ b/optuna_integration/tensorboard.py @@ -3,7 +3,7 @@ import optuna from optuna._experimental import experimental_class -from optuna._imports import try_import +from optuna_integration._imports import try_import from optuna.logging import get_logger diff --git a/tests/test_tensorboard.py b/tests/test_tensorboard.py index 7817c6f6d..54a608e82 100644 --- a/tests/test_tensorboard.py +++ b/tests/test_tensorboard.py @@ -5,8 +5,9 @@ import pytest import optuna -from optuna._imports import try_import -from optuna.integration.tensorboard import TensorBoardCallback + +from optuna_integration._imports import try_import +from optuna_integration.tensorboard import TensorBoardCallback with try_import(): From 726b20e3516a9b294e6eea9eb0d7be0cbeb8f0eb Mon Sep 17 00:00:00 2001 From: dheemantha Date: Mon, 25 Dec 2023 16:19:12 +0100 Subject: [PATCH 5/6] Fixed formatting --- optuna_integration/tensorboard.py | 3 ++- tests/test_tensorboard.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/optuna_integration/tensorboard.py b/optuna_integration/tensorboard.py index e79479eab..5ebd7090a 100644 --- a/optuna_integration/tensorboard.py +++ b/optuna_integration/tensorboard.py @@ -3,9 +3,10 @@ import optuna from optuna._experimental import experimental_class -from optuna_integration._imports import try_import from optuna.logging import get_logger +from optuna_integration._imports import try_import + with try_import() as _imports: from tensorboard.plugins.hparams import api as hp diff --git a/tests/test_tensorboard.py b/tests/test_tensorboard.py index 54a608e82..93d2c3089 100644 --- a/tests/test_tensorboard.py +++ b/tests/test_tensorboard.py @@ -2,9 +2,8 @@ import shutil import tempfile -import pytest - import optuna +import pytest from optuna_integration._imports import try_import from optuna_integration.tensorboard import TensorBoardCallback From d54932bdd0a3a92664242df59648a972293bf31b Mon Sep 17 00:00:00 2001 From: Dheemantha Bhat Date: Tue, 26 Dec 2023 14:42:28 +0100 Subject: [PATCH 6/6] Update docs/source/reference/index.rst Fixed Sphinx warning Co-authored-by: Kento Nozawa --- docs/source/reference/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index 26cd17c6d..10eff59c0 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -94,7 +94,7 @@ skorch optuna.integration.SkorchPruningCallback TensorBoard ----------- +----------- .. autosummary:: :toctree: generated/