Merge branch 'master' of github.com:porink0424/optuna into fix/reduce…

…-select-in-set_trial_param-by-passing-study_id

.github/workflows/tests-with-minimum-versions.yml

@@ -74,7 +74,7 @@ jobs:
         pip uninstall -y matplotlib pandas scipy
         # Python v3.6 was dropped at NumPy v1.20.3.
         if [ "${{ matrix.python-version }}" = "3.7" ]; then
-          pip install matplotlib==3.5.3 pandas==1.3.5 scipy==1.7.3 numpy==1.20.3
+          pip install matplotlib==3.5.3 pandas==1.3.5 scipy==1.7.3 numpy==1.20.3 "fakeredis<2.26.0"
         elif [ "${{ matrix.python-version }}" = "3.8" ]; then
           pip install matplotlib==3.7.5 pandas==2.0.3 scipy==1.10.1 numpy==1.20.3
         elif [ "${{ matrix.python-version }}" = "3.9" ]; then

.github/workflows/tests.yml

@@ -20,7 +20,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
         test-trigger-type:
           - ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && 'Scheduled' || '' }}
         exclude:
@@ -76,6 +76,13 @@ jobs:
 
         pip install --progress-bar off .[test] --extra-index-url https://download.pytorch.org/whl/cpu
         pip install --progress-bar off .[optional] --extra-index-url https://download.pytorch.org/whl/cpu
+        # TODO(c-bata): Remove the version constraint after dropping Python 3.7
+        pip install --progress-bar off "fakeredis<2.26.0"
+
+        # TODO(gen740): Remove this after pytorch supports Python 3.13
+        if [ "${{ matrix.python-version }}" = "3.13" ] ; then
+          pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
+        fi
 
     - name: Output installed packages
       run: |

.pre-commit-config.yaml

@@ -0,0 +1,60 @@
+# pre-commit package installation is necessary to use pre-commit.
+# $ pip install pre-commit
+# $ pre-commit install
+
+default_language_version:
+  python: python3
+
+repos:
+  # Args are based on setup.cfg.
+  - repo: https://github.com/psf/black
+    rev: 24.8.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        exclude: tutorial|docs/visualization_examples|docs/visualization_matplotlib_examples
+        args: [
+            "--max-line-length=99",
+            "--ignore=E203,E704,W503",
+            "--statistics",
+        ]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.11.2
+    hooks:
+    - id: mypy
+      additional_dependencies: [
+        "alembic>=1.5.0",
+        "colorlog",
+        "numpy",
+        "packaging>=20.0",
+        "sqlalchemy>=1.3.0",
+        "tqdm",
+        "PyYAML",
+        "mypy_boto3_s3",
+        "types-PyYAML",
+        "types-redis",
+        "types-setuptools",
+        "types-tqdm",
+        "typing_extensions>=3.10.0.0",
+      ]
+      exclude: docs|tutorial|optuna/storages/_rdb/alembic
+      args: [
+        --warn-unused-configs,
+        --disallow-untyped-calls,
+        --disallow-untyped-defs,
+        --disallow-incomplete-defs,
+        --check-untyped-defs,
+        --no-implicit-optional,
+        --warn-redundant-casts,
+        --strict-equality,
+        --extra-checks,
+        --no-implicit-reexport,
+        --ignore-missing-imports,
+      ]

CONTRIBUTING.md

@@ -78,6 +78,17 @@ $ pip install ".[checking]"
 $ ./formats.sh
 ```
 
+You can use `pre-commit` to automatically check the format, coding style, and type hints before committing.
+The following commands automatically fix format errors by auto-formatters.
+
+```bash
+# Install `pre-commit`.
+$ pip install pre-commit
+
+$ pre-commit install
+$ pre-commit run --all-files
+```
+
 ### Documentation
 
 When adding a new feature to the framework, you also need to document it in the reference.

MANIFEST.in

@@ -1,2 +1,2 @@
-include LICENSE
-include LICENSE_THIRD_PARTY
+graft tests
+global-exclude *~ *.py[cod] *.so

README.md

@@ -25,12 +25,12 @@ Optuna can dynamically construct the search spaces for the hyperparameters.
 
 ## :loudspeaker: News
 <!-- TODO: when you add a new line, please delete the oldest line -->
+* **Oct 21, 2024**: We posted [an article](https://medium.com/optuna/an-introduction-to-moea-d-and-examples-of-multi-objective-optimization-comparisons-8630565a4e89) introducing [MOEA/D](https://hub.optuna.org/samplers/moead/) and an example comparison with other optimization methods.
+* **Oct 15, 2024**: We posted [an article](https://medium.com/optuna/introducing-a-new-terminator-early-termination-of-black-box-optimization-based-on-expected-9a660774fcdb) about `Terminator`, which is expanded in Optuna 4.0.
 * **Sep 18, 2024**: We posted [an article](https://medium.com/optuna/introducing-the-stabilized-journalstorage-in-optuna-4-0-from-mechanism-to-use-case-e320795ffb61) about `JournalStorage`, which is stabilized in Optuna 4.0.
 * **Sep 2, 2024**: Optuna 4.0 is available!  You can install it by `pip install -U optuna`. Find the latest [here](https://github.com/optuna/optuna/releases) and check [our article](https://medium.com/optuna/optuna-4-0-whats-new-in-the-major-release-3325a8420d10).
 * **Aug 30, 2024**: We posted [an article](https://medium.com/optuna/optunahub-a-feature-sharing-platform-for-optuna-now-available-in-official-release-4b99efe9934d) about the official release of [OptunaHub](https://hub.optuna.org/).
 * **Aug 28, 2024**: We posted [an article](https://medium.com/optuna/a-natural-gradient-based-optimization-algorithm-registered-on-optunahub-0dbe17cb0f7d) about [implicit natural gradient optimization (`INGO`)](https://hub.optuna.org/samplers/implicit_natural_gradient/), a sampler newly supported in [OptunaHub](https://hub.optuna.org/).
-* **Aug 20, 2024**: We posted [an article](https://medium.com/optuna/file-management-during-llm-large-language-model-trainings-by-optuna-v4-0-0-artifact-store-5bdd5112f3c7) about [`ArtifactStore`](https://optuna.readthedocs.io/en/stable/reference/artifacts.html) officially supported from Optuna 4.0.
-* **Aug 15, 2024**: We posted [an article](https://medium.com/optuna/introduction-to-catcma-in-optunahub-ffa316309cb8) about [`CatCMA`](https://hub.optuna.org/samplers/catcma/), a sampler newly supported in [OptunaHub](https://hub.optuna.org/).
 
 ## :fire: Key Features
 

docs/source/reference/samplers/index.rst

@@ -20,7 +20,7 @@ The :mod:`~optuna.samplers` module defines a base class for parameter sampling a
 +----------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+-----------------------------------------------------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------------------------------------------------------+
 | Categorical parameters           |:math:`\color{green}\checkmark`|:math:`\color{green}\checkmark`|:math:`\color{green}\checkmark`|    :math:`\blacktriangle`     |                       :math:`\color{green}\checkmark`                       |    :math:`\blacktriangle`     |:math:`\color{green}\checkmark`|    :math:`\blacktriangle`     |                        :math:`\color{green}\checkmark`                        |
 +----------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+-----------------------------------------------------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------------------------------------------------------+
-| Pruning                          |:math:`\color{green}\checkmark`|:math:`\color{green}\checkmark`|:math:`\color{green}\checkmark`|    :math:`\blacktriangle`     |                         :math:`\color{red}\times`                           |:math:`\color{green}\checkmark`|    :math:`\blacktriangle`     |    :math:`\blacktriangle`     |                        :math:`\color{green}\checkmark`                        |
+| Pruning                          |:math:`\color{green}\checkmark`|:math:`\color{green}\checkmark`|:math:`\color{green}\checkmark`|    :math:`\blacktriangle`     |   :math:`\color{red}\times` (:math:`\blacktriangle` for single-objective)   |:math:`\color{green}\checkmark`|    :math:`\blacktriangle`     |    :math:`\blacktriangle`     |                        :math:`\color{green}\checkmark`                        |
 +----------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+-----------------------------------------------------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------------------------------------------------------+
 | Multivariate optimization        |    :math:`\blacktriangle`     |    :math:`\blacktriangle`     |:math:`\color{green}\checkmark`|:math:`\color{green}\checkmark`|                           :math:`\blacktriangle`                            |    :math:`\blacktriangle`     |:math:`\color{green}\checkmark`|:math:`\color{green}\checkmark`|                            :math:`\blacktriangle`                             |
 +----------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+-----------------------------------------------------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------------------------------------------------------+

optuna/_convert_positional_args.py

@@ -80,7 +80,7 @@ def converter_wrapper(*args: Any, **kwargs: Any) -> "_T":
 
             kwargs.update(inferred_kwargs)
 
-            return func(**kwargs)
+            return func(**kwargs)  # type: ignore[call-arg]
 
         return converter_wrapper
 

optuna/_experimental.py

@@ -1,8 +1,9 @@
+from __future__ import annotations
+
 import functools
 import textwrap
 from typing import Any
 from typing import Callable
-from typing import Optional
 from typing import TYPE_CHECKING
 from typing import TypeVar
 import warnings
@@ -49,8 +50,8 @@ def _get_docstring_indent(docstring: str) -> str:
 
 def experimental_func(
     version: str,
-    name: Optional[str] = None,
-) -> "Callable[[Callable[FP, FT]], Callable[FP, FT]]":
+    name: str | None = None,
+) -> Callable[[Callable[FP, FT]], Callable[FP, FT]]:
     """Decorate function as experimental.
 
     Args:
@@ -60,7 +61,7 @@ def experimental_func(
 
     _validate_version(version)
 
-    def decorator(func: "Callable[FP, FT]") -> "Callable[FP, FT]":
+    def decorator(func: Callable[FP, FT]) -> Callable[FP, FT]:
         if func.__doc__ is None:
             func.__doc__ = ""
 
@@ -69,7 +70,7 @@ def decorator(func: "Callable[FP, FT]") -> "Callable[FP, FT]":
         func.__doc__ = func.__doc__.strip() + textwrap.indent(note, indent) + indent
 
         @functools.wraps(func)
-        def wrapper(*args: Any, **kwargs: Any) -> "FT":
+        def wrapper(*args: Any, **kwargs: Any) -> FT:
             warnings.warn(
                 "{} is experimental (supported from v{}). "
                 "The interface can change in the future.".format(
@@ -88,8 +89,8 @@ def wrapper(*args: Any, **kwargs: Any) -> "FT":
 
 def experimental_class(
     version: str,
-    name: Optional[str] = None,
-) -> "Callable[[CT], CT]":
+    name: str | None = None,
+) -> Callable[[CT], CT]:
     """Decorate class as experimental.
 
     Args:
@@ -99,8 +100,8 @@ def experimental_class(
 
     _validate_version(version)
 
-    def decorator(cls: "CT") -> "CT":
-        def wrapper(cls: "CT") -> "CT":
+    def decorator(cls: CT) -> CT:
+        def wrapper(cls: CT) -> CT:
             """Decorates a class as experimental.
 
             This decorator is supposed to be applied to the experimental class.

optuna/storages/_rdb/models.py

@@ -36,8 +36,7 @@
     _Column = mapped_column
 except ImportError:
     # TODO(Shinichi): Remove this after dropping support for SQLAlchemy<2.0.
-    from sqlalchemy import Column as _Column  # type: ignore[assignment]
-
+    from sqlalchemy import Column as _Column  # type: ignore[assignment, no-redef]
 
 # Don't modify this version number anymore.
 # The schema management functionality has been moved to alembic.

optuna/storages/_rdb/storage.py

@@ -588,27 +588,14 @@ def _set_trial_param_without_commit(
         trial = models.TrialModel.find_or_raise_by_id(trial_id, session)
         self.check_trial_is_updatable(trial_id, trial.state)
 
-        trial_param = models.TrialParamModel.find_by_trial_and_param_name(
-            trial, param_name, session
+        trial_param = models.TrialParamModel(
+            trial_id=trial_id,
+            param_name=param_name,
+            param_value=param_value_internal,
+            distribution_json=distributions.distribution_to_json(distribution),
         )
 
-        if trial_param is not None:
-            # Raise error in case distribution is incompatible.
-            distributions.check_distribution_compatibility(
-                distributions.json_to_distribution(trial_param.distribution_json), distribution
-            )
-
-            trial_param.param_value = param_value_internal
-            trial_param.distribution_json = distributions.distribution_to_json(distribution)
-        else:
-            trial_param = models.TrialParamModel(
-                trial_id=trial_id,
-                param_name=param_name,
-                param_value=param_value_internal,
-                distribution_json=distributions.distribution_to_json(distribution),
-            )
-
-            trial_param.check_and_add(session, trial.study_id)
+        trial_param.check_and_add(session, trial.study_id)
 
     def _check_and_set_param_distribution(
         self,

optuna/study/study.py

@@ -1091,7 +1091,7 @@ def _should_skip_enqueue(self, params: Mapping[str, JSONSerializable]) -> bool:
                     if isinstance(param_value, Real)
                     else param_value == existing_param
                 )
-                repeated_params.append(is_repeated)
+                repeated_params.append(bool(is_repeated))
 
             if all(repeated_params):
                 return True

optuna/terminator/improvement/emmr.py

@@ -54,6 +54,12 @@ class EMMREvaluator(BaseImprovementEvaluator):
     - `A stopping criterion for Bayesian optimization by the gap of expected minimum simple
       regrets <https://proceedings.mlr.press/v206/ishibashi23a.html>`__
 
+    Also, there is our blog post explaining this evaluator:
+
+    - `Introducing A New Terminator: Early Termination of Black-box Optimization Based on
+      Expected Minimum Model Regret
+      <https://medium.com/optuna/introducing-a-new-terminator-early-termination-of-black-box-optimization-based-on-expected-9a660774fcdb>`__
+
     Args:
         deterministic_objective:
             A boolean value which indicates whether the objective function is deterministic.

optuna/terminator/improvement/evaluator.py

@@ -122,16 +122,12 @@ class RegretBoundEvaluator(BaseImprovementEvaluator):
     high probability under the Gaussian process model assumption.
 
     Args:
-        gp:
-            A Gaussian process model on which evaluation base. If not specified, the default
-            Gaussian process model is used.
         top_trials_ratio:
             A ratio of top trials to be considered when estimating the regret. Default to 0.5.
         min_n_trials:
             A minimum number of complete trials to estimate the regret. Default to 20.
-        min_lcb_n_additional_samples:
-            A minimum number of additional samples to estimate the lower confidence bound.
-            Default to 2000.
+        seed:
+            Seed for random number generator.
 
     For further information about this evaluator, please refer to the following paper:
 

pyproject.toml

@@ -89,7 +89,7 @@ optional = [
   "scikit-learn>=0.24.2",
   # optuna/visualization/param_importances.py.
   "scipy",  # optuna/samplers/_gp
-  "torch",  # optuna/samplers/_gp
+  "torch; python_version<='3.12'", # TODO(gen740): Remove this line when 'torch', a dependency of 'optuna/_gp', supports Python 3.13
 ]
 test = [
   "coverage",
@@ -98,7 +98,7 @@ test = [
   "moto",
   "pytest",
   "scipy>=1.9.2; python_version>='3.8'",
-  "torch",
+  "torch; python_version<='3.12'", # TODO(gen740): Remove this line when 'torch', a dependency of 'optuna/_gp', supports Python 3.13
 ]
 
 [project.urls]
@@ -127,7 +127,7 @@ version = {attr = "optuna.version.__version__"}
 [tool.black]
 line-length = 99
 target-version = ['py38']
-exclude = '''
+force-exclude = '''
 /(
   \.eggs
   | \.git

tests/storages_tests/test_storages.py

@@ -486,11 +486,6 @@ def test_set_trial_param(storage_mode: str) -> None:
         # Check set_param breaks neither get_trial nor get_trial_params.
         assert storage.get_trial(trial_id_1).params == {"x": 0.5, "y": "Meguro"}
         assert storage.get_trial_params(trial_id_1) == {"x": 0.5, "y": "Meguro"}
-        # Duplicated registration should overwrite.
-        storage.set_trial_param(trial_id_1, "x", 0.6, distribution_x)
-        assert storage.get_trial_param(trial_id_1, "x") == 0.6
-        assert storage.get_trial(trial_id_1).params == {"x": 0.6, "y": "Meguro"}
-        assert storage.get_trial_params(trial_id_1) == {"x": 0.6, "y": "Meguro"}
 
         # Set params to another trial.
         storage.set_trial_param(trial_id_2, "x", 0.3, distribution_x)