Merge branch 'optuna:master' into python3.12

.github/workflows/checks-integration.yml

@@ -33,10 +33,10 @@ jobs:
     - name: Install
       run: |
         python -m pip install -U pip
-        pip install --progress-bar off -U .[benchmark]
+        pip install --progress-bar off -U .[benchmark] --extra-index-url https://download.pytorch.org/whl/cpu
         pip install --progress-bar off -U .[checking]
         pip install --progress-bar off -U .[integration] --extra-index-url https://download.pytorch.org/whl/cpu
-        pip install --progress-bar off -U .[optional]
+        pip install --progress-bar off -U .[optional] --extra-index-url https://download.pytorch.org/whl/cpu
         pip install --progress-bar off -U .[test]
         pip install --progress-bar off -U bayesmark
         pip install --progress-bar off -U kurobako

.github/workflows/coverage.yml

@@ -52,7 +52,7 @@ jobs:
         optuna --version
 
         pip install --progress-bar off .[test]
-        pip install --progress-bar off .[optional]
+        pip install --progress-bar off .[optional] --extra-index-url https://download.pytorch.org/whl/cpu
         pip install --progress-bar off .[integration] --extra-index-url https://download.pytorch.org/whl/cpu
 
         echo 'import coverage; coverage.process_startup()' > sitecustomize.py

.github/workflows/dockerimage.yml

@@ -33,6 +33,8 @@ jobs:
         python_version: ['3.7', '3.8', '3.9', '3.10', '3.11']
         build_type: ['', 'dev']  # "dev" installs all the dependencies including pytest.
         exclude:
+        - python_version: '3.7'
+          build_type: 'dev'
         - python_version: '3.11'
           build_type: 'dev'
 

.github/workflows/performance-benchmarks-bayesmark.yml

@@ -59,7 +59,7 @@ jobs:
         python -c 'import optuna'
         optuna --version
 
-        pip install --progress-bar off .[benchmark]
+        pip install --progress-bar off .[benchmark] --extra-index-url https://download.pytorch.org/whl/cpu
         pip install --progress-bar off bayesmark matplotlib pandas
 
     - name: Output installed packages

.github/workflows/performance-benchmarks-kurobako.yml

@@ -72,7 +72,7 @@ jobs:
         python -c 'import optuna'
         optuna --version
 
-        pip install --progress-bar off .[benchmark]
+        pip install --progress-bar off .[benchmark] --extra-index-url https://download.pytorch.org/whl/cpu
 
         pip install --progress-bar off kurobako
 

.github/workflows/performance-benchmarks-mo-kurobako.yml

@@ -64,7 +64,7 @@ jobs:
         python -c 'import optuna'
         optuna --version
 
-        pip install --progress-bar off .[benchmark]
+        pip install --progress-bar off .[benchmark] --extra-index-url https://download.pytorch.org/whl/cpu
 
         pip install --progress-bar off kurobako
 

.github/workflows/speed-benchmarks.yml

@@ -40,7 +40,7 @@ jobs:
         python -c 'import optuna'
         optuna --version
 
-        pip install --progress-bar off .[benchmark]
+        pip install --progress-bar off .[benchmark] --extra-index-url https://download.pytorch.org/whl/cpu
         asv machine --yes
 
     - name: Output installed packages

.github/workflows/sphinx-build.yml

@@ -38,7 +38,7 @@ jobs:
     - name: Install Dependencies
       run: |
         python -m pip install -U pip
-        pip install --progress-bar off -U .[document]
+        pip install --progress-bar off -U .[document] --extra-index-url https://download.pytorch.org/whl/cpu
 
     - name: Output installed packages
       run: |
@@ -91,7 +91,7 @@ jobs:
     - name: Install Dependencies
       run: |
         python -m pip install -U pip
-        pip install --progress-bar off -U .[document]
+        pip install --progress-bar off -U .[document] --extra-index-url https://download.pytorch.org/whl/cpu
 
     - name: Output installed packages
       run: |

.github/workflows/tests-integration.yml

@@ -56,7 +56,7 @@ jobs:
         optuna --version
 
         pip install --progress-bar off .[test]
-        pip install --progress-bar off .[optional]
+        pip install --progress-bar off .[optional] --extra-index-url https://download.pytorch.org/whl/cpu
         pip install --progress-bar off .[integration] --extra-index-url https://download.pytorch.org/whl/cpu
 
     - name: Output installed packages

.github/workflows/tests-mpi.yml

@@ -55,7 +55,7 @@ jobs:
         optuna --version
 
         pip install --progress-bar off .[test]
-        pip install --progress-bar off .[optional]
+        pip install --progress-bar off .[optional] --extra-index-url https://download.pytorch.org/whl/cpu
 
         # TODO(not522): Remove this line when torchmetrics can be installed with extra-index-url
         pip install --progress-bar off torchmetrics

.github/workflows/tests-storage.yml

@@ -92,7 +92,7 @@ jobs:
         optuna --version
 
         pip install --progress-bar off .[test]
-        pip install --progress-bar off .[optional]
+        pip install --progress-bar off .[optional] --extra-index-url https://download.pytorch.org/whl/cpu
 
     - name: Install DB bindings
       run: |

.github/workflows/tests-with-minimum-versions.yml

@@ -64,7 +64,7 @@ jobs:
         optuna --version
 
         pip install --progress-bar off .[test]
-        pip install --progress-bar off .[optional]
+        pip install --progress-bar off .[optional] --extra-index-url https://download.pytorch.org/whl/cpu
 
     - name: Install dependencies with minimum versions
       run: |

.github/workflows/tests.yml

@@ -64,7 +64,7 @@ jobs:
         optuna --version
 
         pip install --progress-bar off .[test]
-        pip install --progress-bar off .[optional]
+        pip install --progress-bar off .[optional] --extra-index-url https://download.pytorch.org/whl/cpu
 
     - name: Output installed packages
       run: |

docs/source/faq.rst

@@ -619,8 +619,8 @@ will retry failed trials when a new trial starts to evaluate.
     study = optuna.create_study(storage=storage)
 
 
-How can deal with permutation as a parameter?
----------------------------------------------
+How can I deal with permutation as a parameter?
+-----------------------------------------------
 
 Although it is not straightforward to deal with combinatorial search spaces like permutations with existing API, there exists a convenient technique for handling them. 
 It involves re-parametrization of permutation search space of :math:`n` items as an independent :math:`n`-dimensional integer search space. 
@@ -682,3 +682,37 @@ An Optuna implementation example to solve Euclid TSP is as follows:
     study.optimize(objective, n_trials=10)
     lehmer_code = study.best_params.values()
     print(decode(lehmer_code))
+
+How can I ignore duplicated samples?
+------------------------------------
+
+Optuna may sometimes suggest parameters evaluated in the past and if you would like to avoid this problem, you can try out the following workaround:
+
+.. code-block:: python
+
+    import optuna
+    from optuna.trial import TrialState
+
+
+    def objective(trial):
+        # Sample parameters.
+        x = trial.suggest_int("x", -5, 5)
+        y = trial.suggest_int("y", -5, 5)
+        # Fetch all the trials to consider.
+        # In this example, we use only completed trials, but users can specify other states
+        # such as TrialState.PRUNED and TrialState.FAIL.
+        states_to_consider = (TrialState.COMPLETE,)
+        trials_to_consider = trial.study.get_trials(deepcopy=False, states=states_to_consider)
+        # Check whether we already evaluated the sampled `(x, y)`.
+        for t in reversed(trials_to_consider):
+            if trial.params == t.params:
+                # Use the existing value as trial duplicated the parameters.
+                return t.value
+
+        # Compute the objective function if the parameters are not duplicated.
+        # We use the 2D sphere function in this example.
+        return x ** 2 + y ** 2
+    
+
+    study = optuna.create_study()
+    study.optimize(objective, n_trials=100)

formats.sh

@@ -42,7 +42,7 @@ do
   esac
 done
 
-target="optuna tests benchmarks"
+target="optuna tests benchmarks tutorial"
 mypy_target="optuna tests benchmarks"
 res_all=0
 

optuna/integration/_lightgbm_tuner/alias.py

@@ -2,14 +2,15 @@
 
 from collections.abc import Iterable
 from typing import Any
+import warnings
 
 
 _ALIAS_GROUP_LIST: list[dict[str, Any]] = [
     {"param_name": "bagging_fraction", "alias_names": ["sub_row", "subsample", "bagging"]},
     {"param_name": "learning_rate", "alias_names": ["shrinkage_rate", "eta"]},
     {
-        "param_name": "min_data_in_leaf",
-        "alias_names": ["min_data_per_leaf", "min_data", "min_child_samples"],
+        "param_name": "min_child_samples",
+        "alias_names": ["min_data_per_leaf", "min_data", "min_data_in_leaf", "min_samples_leaf"],
     },
     {
         "param_name": "min_sum_hessian_in_leaf",
@@ -20,10 +21,19 @@
             "min_child_weight",
         ],
     },
+    {
+        "param_name": "num_leaves",
+        "alias_names": [
+            "num_leaf",
+            "max_leaves",
+            "max_leaf",
+            "max_leaf_nodes",
+        ],
+    },
     {"param_name": "bagging_freq", "alias_names": ["subsample_freq"]},
     {"param_name": "feature_fraction", "alias_names": ["sub_feature", "colsample_bytree"]},
-    {"param_name": "lambda_l1", "alias_names": ["reg_alpha"]},
-    {"param_name": "lambda_l2", "alias_names": ["reg_lambda", "lambda"]},
+    {"param_name": "lambda_l1", "alias_names": ["reg_alpha", "l1_regularization"]},
+    {"param_name": "lambda_l2", "alias_names": ["reg_lambda", "lambda", "l2_regularization"]},
     {"param_name": "min_gain_to_split", "alias_names": ["min_split_gain"]},
 ]
 
@@ -34,12 +44,21 @@ def _handling_alias_parameters(lgbm_params: dict[str, Any]) -> None:
     for alias_group in _ALIAS_GROUP_LIST:
         param_name = alias_group["param_name"]
         alias_names = alias_group["alias_names"]
+        duplicated_alias: dict[str, Any] = {}
 
         for alias_name in alias_names:
             if alias_name in lgbm_params:
+                duplicated_alias[alias_name] = lgbm_params[alias_name]
                 lgbm_params[param_name] = lgbm_params[alias_name]
                 del lgbm_params[alias_name]
 
+        if len(duplicated_alias) > 1:
+            msg = (
+                f"{param_name} in param detected multiple identical aliases {duplicated_alias}, "
+                f"but we use {param_name}={lgbm_params[param_name]}."
+            )
+            warnings.warn(msg)
+
 
 _ALIAS_METRIC_LIST: list[dict[str, Any]] = [
     # The list `alias_names` do not include the `metric_name` itself.

optuna/integration/_lightgbm_tuner/optimize.py

@@ -394,12 +394,12 @@ def __init__(
         self._model_dir = model_dir
         self._optuna_seed = optuna_seed
 
-        # Should not alter data since `min_data_in_leaf` is tuned.
+        # Should not alter data since `min_child_samples` is tuned.
         # https://lightgbm.readthedocs.io/en/latest/Parameters.html#feature_pre_filter
         if self.lgbm_params.get("feature_pre_filter", False):
             warnings.warn(
                 "feature_pre_filter is given as True but will be set to False. This is required "
-                "for the tuner to tune min_data_in_leaf."
+                "for the tuner to tune min_child_samples."
             )
         self.lgbm_params["feature_pre_filter"] = False
 
@@ -560,7 +560,7 @@ def tune_min_data_in_leaf(self) -> None:
         param_values = [5, 10, 25, 50, 100]
 
         sampler = optuna.samplers.GridSampler({param_name: param_values}, seed=self._optuna_seed)
-        self._tune_params([param_name], len(param_values), sampler, "min_data_in_leaf")
+        self._tune_params([param_name], len(param_values), sampler, "min_child_samples")
 
     def _tune_params(
         self,

optuna/integration/pytorch_distributed.py

@@ -126,9 +126,7 @@ def __init__(
                     raise RuntimeError("torch distributed is not initialized.")
                 default_pg: "ProcessGroup" = dist.group.WORLD
                 if dist.get_backend(default_pg) == "nccl":
-                    new_group: "ProcessGroup" = dist.new_group(  # type: ignore[no-untyped-call]
-                        backend="gloo"
-                    )
+                    new_group: "ProcessGroup" = dist.new_group(backend="gloo")
                     _g_pg = new_group
                 else:
                     _g_pg = default_pg

optuna/samplers/_brute_force.py

@@ -144,9 +144,10 @@ def sample_relative(
         return {}
 
     @staticmethod
-    def _build_tree(trials: Iterable[FrozenTrial], params: Dict[str, Any]) -> _TreeNode:
-        # Build a _TreeNode under given params from the given trials.
-        tree = _TreeNode()
+    def _populate_tree(
+        tree: _TreeNode, trials: Iterable[FrozenTrial], params: Dict[str, Any]
+    ) -> None:
+        # Populate tree under given params from the given trials.
         incomplete_leaves: List[_TreeNode] = []
         for trial in trials:
             if not all(p in trial.params and trial.params[p] == v for p, v in params.items()):
@@ -173,7 +174,6 @@ def _build_tree(trials: Iterable[FrozenTrial], params: Dict[str, Any]) -> _TreeN
         for leaf in incomplete_leaves:
             if leaf.children is None:
                 leaf.set_leaf()
-        return tree
 
     def sample_independent(
         self,
@@ -191,9 +191,13 @@ def sample_independent(
                 TrialState.FAIL,
             ),
         )
-        tree = self._build_tree((t for t in trials if t.number != trial.number), trial.params)
+        tree = _TreeNode()
         candidates = _enumerate_candidates(param_distribution)
         tree.expand(param_name, candidates)
+        # Populating must happen after the initialization above to prevent `tree` from
+        # being initialized as an empty graph, which is created with n_jobs > 1
+        # where we get trials[i].params = {} for some i.
+        self._populate_tree(tree, (t for t in trials if t.number != trial.number), trial.params)
         if tree.count_unexpanded() == 0:
             return param_distribution.to_external_repr(self._rng.rng.choice(candidates))
         else:
@@ -215,7 +219,9 @@ def after_trial(
                 TrialState.FAIL,
             ),
         )
-        tree = self._build_tree(
+        tree = _TreeNode()
+        self._populate_tree(
+            tree,
             (
                 t
                 if t.number != trial.number

optuna/samplers/_tpe/sampler.py

@@ -565,16 +565,16 @@ def after_trial(
         self._random_sampler.after_trial(study, trial, state, values)
 
 
-def _calculate_nondomination_rank(loss_vals: np.ndarray) -> np.ndarray:
+def _calculate_nondomination_rank(loss_vals: np.ndarray, n_below: int) -> np.ndarray:
     ranks = np.full(len(loss_vals), -1)
-    num_unranked = len(loss_vals)
+    num_ranked = 0
     rank = 0
     domination_mat = np.all(loss_vals[:, None, :] >= loss_vals[None, :, :], axis=2) & np.any(
         loss_vals[:, None, :] > loss_vals[None, :, :], axis=2
     )
-    while num_unranked > 0:
+    while num_ranked < n_below:
         counts = np.sum((ranks == -1)[None, :] & domination_mat, axis=1)
-        num_unranked -= np.sum((counts == 0) & (ranks == -1))
+        num_ranked += np.sum((counts == 0) & (ranks == -1))
         ranks[(counts == 0) & (ranks == -1)] = rank
         rank += 1
     return ranks
@@ -654,7 +654,7 @@ def _split_complete_trials_multi_objective(
             lvals[:, i] *= -1
 
     # Solving HSSP for variables number of times is a waste of time.
-    nondomination_ranks = _calculate_nondomination_rank(lvals)
+    nondomination_ranks = _calculate_nondomination_rank(lvals, n_below)
     assert 0 <= n_below <= len(lvals)
 
     indices = np.array(range(len(lvals)))

optuna/version.py

@@ -1 +1 @@
-__version__ = "3.4.0.dev"
+__version__ = "3.5.0.dev"

optuna/visualization/_parallel_coordinate.py

@@ -214,7 +214,9 @@ def _target(t: FrozenTrial) -> float:
             values = [math.log10(v) for v in values]
             min_value = min(values)
             max_value = max(values)
-            tickvals: list[int | float] = list(range(math.ceil(min_value), math.ceil(max_value)))
+            tickvals: list[int | float] = list(
+                range(math.ceil(min_value), math.floor(max_value) + 1)
+            )
             if min_value not in tickvals:
                 tickvals = [min_value] + tickvals
             if max_value not in tickvals: