diff --git a/package/samplers/differential_evolution/LICENSE b/package/samplers/differential_evolution/LICENSE new file mode 100644 index 00000000..8d1287a5 --- /dev/null +++ b/package/samplers/differential_evolution/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jinglue Xu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/package/samplers/differential_evolution/README.md b/package/samplers/differential_evolution/README.md new file mode 100644 index 00000000..1dc39cd1 --- /dev/null +++ b/package/samplers/differential_evolution/README.md @@ -0,0 +1,162 @@ +--- +author: Jinglue Xu +title: Differential Evolution Sampler +description: This sampler combines Differential Evolution for numerical parameters and Random Sampling for categorical parameters, dynamically adapting to changes in the search space by initializing new dimensions and ignoring removed ones seamlessly across trials. +tags: [sampler, differential evolution, dynamic search space, mixed-variable optimization] +optuna_versions: [4.1.0] +license: MIT License +--- + +## Abstract + +### Differential Evolution (DE) Sampler + +This implementation introduces a novel Differential Evolution (DE) sampler, tailored to optimize both numerical and categorical hyperparameters effectively. The DE sampler integrates a hybrid approach: + +1. **Differential Evolution for Numerical Parameters:** Exploiting DE’s strengths, the sampler efficiently explores numerical parameter spaces through mutation, crossover, and selection mechanisms. +1. **Random Sampling for Categorical Parameters:** For categorical variables, the sampler employs random sampling, ensuring comprehensive coverage of discrete spaces. + +The sampler also supports **dynamic search spaces**, enabling seamless adaptation to varying parameter dimensions during optimization. To maintain diversity and scalability, the population size is adaptively determined based on the search space dimensionality. + +### Performance Verification + +The sampler's performance was validated using four standard optimization benchmarks: + +- **Ackley function (Minimization)** +- **Rastrigin function (Minimization)** +- **Sphere function (Minimization)** +- **Schwefel function (Maximization)** + +Each benchmark was tested across 10 experiments. The results demonstrate superior performance in convergence speed and objective value minimization/maximization compared to a random sampling baseline. + +The plots below illustrate the comparative performance, showcasing both mean performance and standard deviation for the DE and random samplers across trial numbers. + + + + + + + + + + +
Top LeftTop Right
Bottom LeftBottom Right
+ +## APIs + +### Differential Evolution (DE) Sampler API Documentation + +The `DESampler` is a hybrid sampler designed to optimize both numerical and categorical hyperparameters efficiently. It combines Differential Evolution (DE) for numerical parameter optimization and random sampling for categorical parameters, making it versatile and scalable for various optimization tasks. + +______________________________________________________________________ + +### Class: `DESampler` + +```python + DESampler( + search_space: dict[str, optuna.distributions.BaseDistribution] | None = None, + population_size: int | str = "auto", + F: float = 0.8, + CR: float = 0.7, + debug: bool = False, + seed: int | None = None + ) +``` + +### Parameters + +#### `search_space` + +A dictionary containing the search space that defines the parameter space. The keys are parameter names, and the values are [Optuna distributions](https://optuna.readthedocs.io/en/stable/reference/distributions.html) specifying the parameter ranges. + +**Example**: + +```python +search_space = { + "x": optuna.distributions.FloatDistribution(-5, 5), + "y": optuna.distributions.FloatDistribution(-5, 5), + "z": optuna.distributions.CategoricalDistribution([0, 1, 2]), +} +sampler = DESampler(search_space=search_space) +``` + +______________________________________________________________________ + +#### `population_size` + +The number of individuals in the population. If set to `"auto"`, the population size is dynamically determined based on the dimensionality of the search space. You can specify a custom integer value for precise control over population size. + +- **Default**: `"auto"` +- **Example**: `population_size=50` + +______________________________________________________________________ + +#### `F` + +Mutation scaling factor. Controls the amplification of the difference between two individuals in DE. + +- **Default**: `0.8` +- **Range**: `[0.0, 2.0]` + +______________________________________________________________________ + +#### `CR` + +Crossover probability. Controls the fraction of parameter values copied from the mutant during crossover. + +- **Default**: `0.7` +- **Range**: `[0.0, 1.0]` + +______________________________________________________________________ + +#### `debug` + +A toggle to enable or disable debug messages for performance monitoring and troubleshooting. + +- **Default**: `False` +- **Example**: `debug=True` + +______________________________________________________________________ + +#### `seed` + +Seed for the random number generator, ensuring reproducibility of results. + +- **Default**: `None` +- **Example**: `seed=42` + +## Installation + +No additional packages besides `optuna` and `optunahub` are required. + +## Example + +```python +import optuna +import math +import optunahub + + +# Define the Rastrigin objective function +def objective_rastrigin(trial): + n_dimensions = 10 # Dimensionality of the problem + variables = [trial.suggest_float(f"x{i}", -5.12, 5.12) for i in range(n_dimensions)] + A = 10 + result = A * n_dimensions + sum(x**2 - A * math.cos(2 * math.pi * x) for x in variables) + return result + +# Initialize the DE Sampler +module = optunahub.load_module("samplers/differential_evolution") +DESampler = module.DESampler +sampler = DESampler(population_size="auto", F=0.8, CR=0.9, seed=42) + +# Create and optimize the study +study = optuna.create_study(direction="minimize", sampler=sampler) +study.optimize(objective_rastrigin, n_trials=10000) + +# Print the results +print("Best parameters:", study.best_params) +print("Best value:", study.best_value) +``` + +For a comprehensive example with benchmarking, see [example.py](https://github.com/optuna/optunahub-registry/blob/main/package/samplers/differential_evolution/example.py). diff --git a/package/samplers/differential_evolution/__init__.py b/package/samplers/differential_evolution/__init__.py new file mode 100644 index 00000000..3ab09ea8 --- /dev/null +++ b/package/samplers/differential_evolution/__init__.py @@ -0,0 +1,4 @@ +from .de import DESampler + + +__all__ = ["DESampler"] diff --git a/package/samplers/differential_evolution/de.py b/package/samplers/differential_evolution/de.py new file mode 100644 index 00000000..7a748e25 --- /dev/null +++ b/package/samplers/differential_evolution/de.py @@ -0,0 +1,478 @@ +from __future__ import annotations + +import time +from typing import Any + +import numpy as np +import optuna +from optuna.samplers import RandomSampler +import optunahub + + +class DESampler(optunahub.samplers.SimpleBaseSampler): + """Differential Evolution Sampler with Random Sampling for categorical parameters. + + This implements a hybrid sampling approach that: + 1. Uses DE algorithm for numerical parameters (float, int). + 2. Uses Random Sampling for categorical parameters. + 3. Combines both sampling strategies seamlessly. + + This also handles dynamic search space for numerical dimensions by: + - For added dimensions in a trial: + - Generation 0 (Random Sampling): + The value for a new dimension is directly initialized by random sampling within the parameter's range. + - Subsequent Generations (Differential Evolution): + The new dimensions are initialized for the sampled individuals (r1, r2, r3) in the trial using the mean + of the parameter's range. If the new dimension persists in subsequent trials, its values for the sampled + individual in the trial are kept for subsequent trials. + - For removed dimensions in a trial: + Simply ignore the dimensions along with their values for all individuals. + + Args: + search_space: + Dictionary mapping parameter names to their distribution ranges. + population_size: + Number of individuals in the population. + F: + Mutation scaling factor - controls the amplification of differential evolution. + CR: + Crossover probability - controls the fraction of parameter values copied from the mutant. + debug: + Toggle for debug messages. + seed: + Random seed for reproducibility. + + Attributes: + seed: + Random seed for reproducibility. + _rng: + Random state object for sampling. + _random_sampler: + Random sampler instance for categorical parameters. + population_size: + Number of individuals in the population. + F: + Mutation scaling factor for DE. + CR: + Crossover probability for DE. + debug: + Debugging toggle. + dim: + Dimensionality of the search space. + population: + Population array for DE sampling. + fitness: + Fitness values of the population. + trial_vectors: + Trial vectors generated for a generation. + lower_bound: + Lower bounds of the numerical parameters. + upper_bound: + Upper bounds of the numerical parameters. + numerical_params: + List of numerical parameter names. + categorical_params: + List of categorical parameter names. + last_time: + Timestamp of the last performance measurement. + last_trial_count: + Count of trials completed at the last performance measurement. + last_processed_gen: + Last processed generation. + current_gen_vectors: + Trial vectors for the current generation. + """ + + def __init__( + self, + search_space: dict[str, optuna.distributions.BaseDistribution] | None = None, + population_size: int | str = "auto", + F: float = 0.8, + CR: float = 0.7, + debug: bool = False, + seed: int | None = None, + ) -> None: + """Initialize the DE sampler.""" + super().__init__(search_space) + + # Store and set random seed + self.seed = seed + self._rng = np.random.RandomState(seed) + + # Initialize random sampler for categorical parameters + self._random_sampler = RandomSampler(seed=seed) + + # DE algorithm parameters + self.population_size = population_size + self.F = F + self.CR = CR + self.debug = debug + + # Search space parameters + self.dim = 0 + self.population: np.ndarray | None = None + self.fitness: np.ndarray | None = None + self.trial_vectors: np.ndarray | None = None + self.lower_bound: np.ndarray | None = None + self.upper_bound: np.ndarray | None = None + + # Parameter type tracking + self.numerical_params: list[str] = [] + self.categorical_params: list[str] = [] + + # Performance tracking + self.last_time = time.time() + self.last_trial_count = 0 + + # Generation management + self.last_processed_gen = -1 + self.current_gen_vectors: np.ndarray | None = None + + if self.population_size == "auto": + self.population_size = self._determine_pop_size(search_space) + + def _determine_pop_size( + self, search_space: dict[str, optuna.distributions.BaseDistribution] | None + ) -> int: + """Determine the population size based on the search space dimensionality. + + Args: + search_space: + Dictionary mapping parameter names to their distribution ranges. + + Returns: + int: + The population size. + """ + if search_space is None: + return 20 + else: + dimension = len(search_space) + + # Start with a baseline multiplier + if dimension < 5: + # For very low dimension, maintain at least 20 individuals + # to ensure diversity. + base_multiplier = 10 + min_pop = 20 + elif dimension <= 30: + # For moderately sized problems, a standard 10x dimension + # is a good starting point. + base_multiplier = 10 + min_pop = 30 + else: + # For high-dimensional problems, start lower (5x) + # to keep computations manageable. + base_multiplier = 5 + min_pop = 50 + + # Calculate a preliminary population size (can be fine-tuned further) + population_size = max(min_pop, base_multiplier * dimension) + + return population_size + + def _split_search_space( + self, search_space: dict[str, optuna.distributions.BaseDistribution] + ) -> tuple[dict, dict]: + """Split search space into numerical and categorical parameters. + + Args: + search_space: + Complete search space dictionary. + + Returns: + tuple: + A tuple of dictionaries (numerical_space, categorical_space). + """ + numerical_space = {} + categorical_space = {} + + for name, dist in search_space.items(): + if isinstance( + dist, + (optuna.distributions.FloatDistribution, optuna.distributions.IntDistribution), + ): + numerical_space[name] = dist + else: + categorical_space[name] = dist + + return numerical_space, categorical_space + + def _generate_trial_vectors(self, active_indices: list[int]) -> np.ndarray: + """Generate new trial vectors using DE mutation and crossover. + + Args: + active_indices: + Indices of active dimensions in the current trial's search space. + + Returns: + np.ndarray: + Array of trial vectors (population_size x len(active_indices)). + """ + + if isinstance(self.population_size, str): + raise ValueError("Population size must be resolved to an integer before this point.") + + trial_vectors = np.zeros((self.population_size, len(active_indices))) + + for i in range(self.population_size): + # Select three random distinct individuals for mutation + indices = [idx for idx in range(self.population_size) if idx != i] + r1, r2, r3 = self._rng.choice(indices, 3, replace=False) + + if self.population is None or self.lower_bound is None or self.upper_bound is None: + raise ValueError( + "Population, lower_bound, and upper_bound must be initialized before this operation." + ) + + # Handle NaN values by filling with default (mean of bounds) + valid_population = np.nan_to_num( + self.population[:, active_indices], + nan=(self.lower_bound[active_indices] + self.upper_bound[active_indices]) / 2, + ) + + # Mutation: v = x_r1 + F * (x_r2 - x_r3) for active indices only + mutant = valid_population[r1] + self.F * (valid_population[r2] - valid_population[r3]) + # Clip mutant vector to bounds for active dimensions + mutant = np.clip( + mutant, self.lower_bound[active_indices], self.upper_bound[active_indices] + ) + + # Crossover: combine target vector with mutant vector + trial = np.copy(valid_population[i]) + crossover_mask = self._rng.rand(len(active_indices)) < self.CR + + # Ensure at least one parameter is taken from mutant vector + if not np.any(crossover_mask): + crossover_mask[self._rng.randint(len(active_indices))] = True + trial[crossover_mask] = mutant[crossover_mask] + + trial_vectors[i] = trial + + return trial_vectors + + def _debug_print(self, message: str) -> None: + """Print debug message if debug mode is enabled. + + Args: + message: + The message to print. + """ + if self.debug: + print(message) + + def _calculate_speed(self, n_completed: int) -> None: + """Calculate and print optimization speed every 100 trials. + + Args: + n_completed: + The number of completed trials. + """ + if not self.debug: + return + + if n_completed % 100 == 0 and n_completed > 0: + current_time = time.time() + elapsed_time = current_time - self.last_time + trials_processed = n_completed - self.last_trial_count + + if elapsed_time > 0: + speed = trials_processed / elapsed_time + print(f"\n[Speed Stats] Trials {self.last_trial_count} to {n_completed}") + print(f"Speed: {speed:.2f} trials/second") + print(f"Time elapsed: {elapsed_time:.2f} seconds") + print("-" * 50) + + self.last_time = current_time + self.last_trial_count = n_completed + + def _get_generation_trials(self, study: optuna.study.Study, generation: int) -> list: + """Get all completed trials for a specific generation. + + Args: + study: + Optuna study object. + generation: + The generation number. + + Returns: + list: + A list of completed trials for the specified generation. + """ + all_trials = study.get_trials(deepcopy=False) + return [ + t + for t in all_trials + if ( + t.state == optuna.trial.TrialState.COMPLETE + and t.system_attrs.get("differential_evolution:generation") == generation + ) + ] + + def sample_relative( + self, + study: optuna.study.Study, + trial: optuna.trial.FrozenTrial, + search_space: dict[str, optuna.distributions.BaseDistribution], + ) -> dict[str, Any]: + """Sample parameters for a trial using hybrid DE/Random sampling approach. + + Args: + study: + Optuna study object. + trial: + Current trial object. + search_space: + Dictionary of parameter distributions. + + Returns: + dict: + A dictionary of parameter values for the trial. + """ + if len(search_space) == 0: + return {} + + # Determine the direction of optimization + sign = 1 if study.direction == optuna.study.StudyDirection.MINIMIZE else -1 + + # Split search space into numerical and categorical + numerical_space, categorical_space = self._split_search_space(search_space) + + # Sample categorical parameters using random sampler + categorical_params = {} + for param_name, distribution in categorical_space.items(): + categorical_params[param_name] = self._random_sampler.sample_independent( + study, trial, param_name, distribution + ) + + # If no numerical parameters, return only categorical + if not numerical_space: + return categorical_params + + # Track active dimensions for the current trial + active_keys = list(numerical_space.keys()) + + # Ensure numerical_params includes all possible keys + if self.numerical_params is None: + self.numerical_params = active_keys + else: + # Dynamically adjust numerical_params to reflect the current trial's search space + for key in active_keys: + if key not in self.numerical_params: + self.numerical_params.append(key) + + # Get indices for the active keys + active_indices = [self.numerical_params.index(name) for name in active_keys] + + if not isinstance(self.population_size, int): + raise ValueError( + "Population size must be an integer before initializing trial vectors." + ) + + # Calculate current generation and individual index + current_generation = trial._trial_id // self.population_size + individual_index = trial._trial_id % self.population_size + + # Store generation and individual info as trial attributes + study._storage.set_trial_system_attr( + trial._trial_id, "differential_evolution:generation", current_generation + ) + study._storage.set_trial_system_attr( + trial._trial_id, "differential_evolution:individual", individual_index + ) + + self._calculate_speed(trial._trial_id) + + # Initialize population and bounds for the entire search space if not done + if self.population is None: + self._debug_print("\nInitializing population...") + all_keys = list(numerical_space.keys()) + self.lower_bound = np.asarray([dist.low for dist in numerical_space.values()]) + self.upper_bound = np.asarray([dist.high for dist in numerical_space.values()]) + self.dim = len(all_keys) + + # Initialize population using seeded RNG + self.population = ( + self._rng.rand(self.population_size, self.dim) + * (self.upper_bound - self.lower_bound) + + self.lower_bound + ) + self.fitness = np.full(self.population_size, -np.inf if sign == -1 else np.inf) + self.numerical_params = all_keys # Track all keys + + # Initial population evaluation + if current_generation == 0: + self._debug_print( + f"Evaluating initial individual {individual_index + 1}/{self.population_size}" + ) + numerical_params = { + name: ( + float(value) + if isinstance(numerical_space[name], optuna.distributions.FloatDistribution) + else int(value) + ) + for name, value in zip( + active_keys, self.population[individual_index, active_indices] + ) + } + return {**numerical_params, **categorical_params} + + # Process previous generation if needed + if current_generation > 0 and current_generation != self.last_processed_gen: + prev_gen = current_generation - 1 + prev_trials = self._get_generation_trials(study, prev_gen) + + if len(prev_trials) == self.population_size: + self._debug_print(f"\nProcessing generation {prev_gen}") + + # Get fitness and parameter values from previous generation + trial_fitness = np.array([sign * t.value for t in prev_trials]) + + # Initialize trial_vectors with uniform size, using NaN or a default value for missing parameters + trial_vectors = np.full( + (self.population_size, len(self.numerical_params)), + np.nan, # Placeholder for missing parameters + ) + + for i, t in enumerate(prev_trials): + for j, name in enumerate(self.numerical_params): + if name in t.params: # Only include active parameters + trial_vectors[i, j] = t.params[name] + + # if not isinstance(self.population_size, int): + # raise ValueError("Population size must be an integer before this point.") + + if self.fitness is None: + raise ValueError("Fitness array must be initialized before this operation.") + + if trial_fitness is None: + raise ValueError( + "Trial fitness array must be initialized before this operation." + ) + + # Selection: keep better solutions + for i in range(self.population_size): + if trial_fitness[i] <= sign * self.fitness[i]: + self.population[i, active_indices] = trial_vectors[i, active_indices] + self.fitness[i] = sign * trial_fitness[i] + + self._debug_print(f"Best fitness: {np.nanmin(sign * self.fitness):.6f}") + + # Generate new trial vectors for current generation + self.current_gen_vectors = self._generate_trial_vectors(active_indices) + self.last_processed_gen = current_generation + + # Ensure we have trial vectors for current generation + if self.current_gen_vectors is None: + self.current_gen_vectors = self._generate_trial_vectors(active_indices) + + # Combine numerical and categorical parameters + numerical_params = { + name: ( + float(value) + if isinstance(numerical_space[name], optuna.distributions.FloatDistribution) + else int(value) + ) + for name, value in zip(active_keys, self.current_gen_vectors[individual_index]) + } + return {**numerical_params, **categorical_params} diff --git a/package/samplers/differential_evolution/example.py b/package/samplers/differential_evolution/example.py new file mode 100644 index 00000000..6c5af522 --- /dev/null +++ b/package/samplers/differential_evolution/example.py @@ -0,0 +1,430 @@ +""" +To run this example, you need to install the following packages: + +matplotlib>=3.9.2 +numpy>=2.1.3 +scipy>=1.14.1 +scikit-learn>=1.5.2 + +""" + +from __future__ import annotations + +import math +import os + +import matplotlib.pyplot as plt +import numpy as np +import optuna +import optunahub +from sklearn.datasets import load_digits +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import cross_val_score +from sklearn.model_selection import train_test_split +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler + + +# ---------------Objective Functions--------------- + + +def objective_Ackley(trial: optuna.Trial) -> float: + """Ackley function optimization. + + Args: + trial: + The trial object to suggest parameters. + + Returns: + The computed Ackley function value. + """ + n_dimensions = 10 # High-dimensional problem with 10 dimensions + + # Suggest a value for each dimension + variables = [trial.suggest_float(f"x{i}", -32.768, 32.768) for i in range(n_dimensions)] + + # Ackley function parameters + a = 20 + b = 0.2 + c = 2 * math.pi + + # Compute the function components + sum_sq_term = sum(x**2 for x in variables) + cos_term = sum(math.cos(c * x) for x in variables) + + # Return the function value + return ( + -a * math.exp(-b * math.sqrt(sum_sq_term / n_dimensions)) + - math.exp(cos_term / n_dimensions) + + a + + math.exp(1) + ) + + +def objective_sphere(trial: optuna.Trial) -> float: + """Sphere function optimization. + + Args: + trial: + The trial object to suggest parameters. + + Returns: + The computed Sphere function value. + """ + n_dimensions = 10 # Number of dimensions + + # Suggest a value for each dimension + variables = [trial.suggest_float(f"x{i}", -10.0, 10.0) for i in range(n_dimensions)] + + # Return the sum of squares + return sum(x**2 for x in variables) + + +def objective_Rastrigin(trial: optuna.Trial) -> float: + """Rastrigin function optimization. + + Args: + trial: + The trial object to suggest parameters. + + Returns: + The computed Rastrigin function value. + """ + n_dimensions = 10 # High-dimensional problem with 10 dimensions + variables = [trial.suggest_float(f"x{i}", -5.12, 5.12) for i in range(n_dimensions)] + A = 10 + + # Compute the Rastrigin function value + sum_term = sum(x**2 - A * math.cos(2 * math.pi * x) for x in variables) + return A * n_dimensions + sum_term + + +def objective_Schwefel(trial: optuna.Trial) -> float: + """Schwefel function optimization. + + Args: + trial: + The trial object to suggest parameters. + + Returns: + The computed Schwefel function value. + """ + n_dimensions = 10 + variables = [trial.suggest_float(f"x{i}", -500, 500) for i in range(n_dimensions)] + + # Compute the Schwefel function value + sum_term = sum(x * math.sin(math.sqrt(abs(x))) for x in variables) + return -(418.9829 * n_dimensions - sum_term) + 10000 + + +def objective_ML(trial: optuna.Trial) -> float: + """Machine learning objective using RandomForestClassifier. + + Args: + trial: + The trial object to suggest hyperparameters. + + Returns: + Mean accuracy obtained using cross-validation. + """ + # Load dataset + data = load_digits() + X, y = data.data, data.target + + # Split data into training and testing sets + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42) + + # Hyperparameter suggestions + n_estimators = trial.suggest_int("n_estimators", 50, 300) + max_depth = trial.suggest_int("max_depth", 5, 30) + min_samples_split = trial.suggest_int("min_samples_split", 2, 15) + min_samples_leaf = trial.suggest_int("min_samples_leaf", 1, 10) + max_features = trial.suggest_categorical("max_features", ["sqrt", "log2", None]) + bootstrap = trial.suggest_categorical("bootstrap", [True, False]) + + # Define a pipeline with scaling and RandomForestClassifier + pipeline = Pipeline( + [ + ("scaler", StandardScaler()), + ( + "classifier", + RandomForestClassifier( + n_estimators=n_estimators, + max_depth=max_depth, + min_samples_split=min_samples_split, + min_samples_leaf=min_samples_leaf, + max_features=max_features, + bootstrap=bootstrap, + random_state=42, + ), + ), + ] + ) + + # Cross-validation for accuracy + scores = cross_val_score(pipeline, X_train, y_train, cv=5, scoring="accuracy") + return scores.mean() + + +# ruff: noqa +def objective_dynamic_1(trial: optuna.Trial) -> float: + """Dynamic search space function with a single additional parameter for the first trial. + + Args: + trial: + The trial object to suggest parameters. + + Returns: + Computed value of the function. + """ + x = trial.suggest_float("x", -5.12, 5.12) + y = trial.suggest_float("y", -5.12, 5.12) + if trial.number == 0: + z = trial.suggest_float("z", -5.12, 5.12) + return x**2 + y**2 + + +def objective_dynamic_2(trial: optuna.Trial) -> float: + """Dynamic search space function with a single additional parameter at the 100th trial. + + Args: + trial: + The trial object to suggest parameters. + + Returns: + Computed value of the function. + """ + x = trial.suggest_float("x", -5.12, 5.12) + y = trial.suggest_float("y", -5.12, 5.12) + if trial.number == 100: + z = trial.suggest_float("z", -5.12, 5.12) + return x**2 + y**2 + + +def objective_dynamic_3(trial: optuna.Trial) -> float: + """Dynamic search space function with varying parameters at specific trial numbers. + + Args: + trial: + The trial object to suggest parameters. + + Returns: + Computed value of the function. + """ + x = trial.suggest_float("x", -5.12, 5.12) + y = trial.suggest_float("y", -5.12, 5.12) + if trial.number == 0: + z = trial.suggest_float("z", -5.12, 5.12) + if 100 <= trial.number < 200: + z = trial.suggest_float("z", -5.12, 5.12) + if trial.number == 300: + z = trial.suggest_float("z", -5.12, 5.12) + return x**2 + y**2 + + +# ruff: enable + +# Mapping of objective functions +objective_map = { + "Ackley": objective_Ackley, + "sphere": objective_sphere, + "Rastrigin": objective_Rastrigin, + "Schwefel": objective_Schwefel, + "ML": objective_ML, + "dynamic_1": objective_dynamic_1, + "dynamic_2": objective_dynamic_2, + "dynamic_3": objective_dynamic_3, +} + +# ---------------Settings--------------- + +# Toggle for running the benchmark +run_benchmark = True + +# Choose a specific objective function for single experiment runs +objective_function_choice = "Rastrigin" +# Options: "Ackley", "sphere", "Rastrigin", "Schwefel", "ML", "dynamic_1", "dynamic_2", "dynamic_3" + +# DE Sampler settings +population_size = "auto" +F = 0.8 +CR = 0.9 +debug = True + +# Experiment configuration +num_experiments = 10 # Number of independent experiments +number_of_trials = 10000 # Number of trials per experiment + +# ---------------Experiments--------------- + +if run_benchmark: + # Run the benchmark for multiple objective functions + + # Ensure the results directory exists + os.makedirs("results", exist_ok=True) + + # List of objective functions to evaluate + objective_list = ["Ackley", "sphere", "Rastrigin", "Schwefel"] + + for objective_function_choice in objective_list: + # Determine optimization direction + direction = "maximize" if objective_function_choice == "Schwefel" else "minimize" + minimize = direction == "minimize" + + # Get the mapped objective function + objective_function = objective_map[objective_function_choice] + + # Load the DE Sampler + sampler = optunahub.load_module("samplers/differential_evolution").DESampler( + population_size=population_size, F=F, CR=CR, debug=debug + ) + + # Load the Random Sampler + sampler_rs = optuna.samplers.RandomSampler(seed=42) + + # Initialize result storage + results_de = np.zeros((num_experiments, number_of_trials)) + results_rs = np.zeros((num_experiments, number_of_trials)) + + # Run experiments for the DE Sampler and Random Sampler + for i in range(num_experiments): + # Run DE Sampler + study = optuna.create_study(sampler=sampler, direction=direction) + study.optimize(objective_function, n_trials=number_of_trials, n_jobs=16) + + # Track DE Sampler's best values + best_values_de = [] + current_best_de = float("inf") if minimize else float("-inf") + for trial in study.trials: + if trial.value is not None: + current_best_de = ( + min(current_best_de, trial.value) + if minimize + else max(current_best_de, trial.value) + ) + best_values_de.append(current_best_de) + results_de[i, :] = best_values_de + + # Run Random Sampler + study_rs = optuna.create_study(sampler=sampler_rs, direction=direction) + study_rs.optimize(objective_function, n_trials=number_of_trials, n_jobs=16) + + # Track Random Sampler's best values + best_values_rs = [] + current_best_rs = float("inf") if minimize else float("-inf") + for trial in study_rs.trials: + if trial.value is not None: + current_best_rs = ( + min(current_best_rs, trial.value) + if minimize + else max(current_best_rs, trial.value) + ) + best_values_rs.append(current_best_rs) + results_rs[i, :] = best_values_rs + + # Compute and plot performance metrics + mean_de = np.mean(results_de, axis=0) + std_de = np.std(results_de, axis=0) + mean_rs = np.mean(results_rs, axis=0) + std_rs = np.std(results_rs, axis=0) + + plt.figure(figsize=(10, 6)) + plt.plot(mean_de, label="DESampler (Mean Performance)", linestyle="-", color="blue") + plt.fill_between( + range(number_of_trials), mean_de - std_de, mean_de + std_de, color="blue", alpha=0.2 + ) + plt.plot(mean_rs, label="RandomSampler (Mean Performance)", linestyle="--", color="orange") + plt.fill_between( + range(number_of_trials), mean_rs - std_rs, mean_rs + std_rs, color="orange", alpha=0.2 + ) + plt.title( + f"Performance Comparison ({objective_function_choice.capitalize()} - {direction.capitalize()})" + ) + plt.xlabel("Trial Number") + plt.ylabel("Objective Value (Log Scale)") + plt.yscale("log") + plt.grid(which="both", linestyle="--", linewidth=0.5) + plt.legend() + + # Save plot to file + filename = f"results/{objective_function_choice}_{direction}.png" + plt.savefig(filename, dpi=300) + plt.show() + +else: + # Run a single experiment for the chosen objective function + + direction = "maximize" if objective_function_choice == "Schwefel" else "minimize" + minimize = direction == "minimize" + + # Get the mapped objective function + objective_function = objective_map[objective_function_choice] + + # Load the DE Sampler + sampler = optunahub.load_module("samplers/differential_evolution").DESampler( + population_size=population_size, F=F, CR=CR, debug=debug + ) + + # Load the Random Sampler + sampler_rs = optuna.samplers.RandomSampler(seed=42) + + results_de = np.zeros((num_experiments, number_of_trials)) + results_rs = np.zeros((num_experiments, number_of_trials)) + + for i in range(num_experiments): + # Run DE Sampler + study = optuna.create_study(sampler=sampler, direction=direction) + study.optimize(objective_function, n_trials=number_of_trials, n_jobs=16) + + best_values_de = [] + current_best_de = float("inf") if minimize else float("-inf") + for trial in study.trials: + if trial.value is not None: + current_best_de = ( + min(current_best_de, trial.value) + if minimize + else max(current_best_de, trial.value) + ) + best_values_de.append(current_best_de) + results_de[i, :] = best_values_de + + # Run Random Sampler + study_rs = optuna.create_study(sampler=sampler_rs, direction=direction) + study_rs.optimize(objective_function, n_trials=number_of_trials, n_jobs=16) + + best_values_rs = [] + current_best_rs = float("inf") if minimize else float("-inf") + for trial in study_rs.trials: + if trial.value is not None: + current_best_rs = ( + min(current_best_rs, trial.value) + if minimize + else max(current_best_rs, trial.value) + ) + best_values_rs.append(current_best_rs) + results_rs[i, :] = best_values_rs + + # Compute and display performance metrics + mean_de = np.mean(results_de, axis=0) + std_de = np.std(results_de, axis=0) + mean_rs = np.mean(results_rs, axis=0) + std_rs = np.std(results_rs, axis=0) + + plt.figure(figsize=(10, 6)) + plt.plot(mean_de, label="DESampler (Mean Performance)", linestyle="-", color="blue") + plt.fill_between( + range(number_of_trials), mean_de - std_de, mean_de + std_de, color="blue", alpha=0.2 + ) + plt.plot(mean_rs, label="RandomSampler (Mean Performance)", linestyle="--", color="orange") + plt.fill_between( + range(number_of_trials), mean_rs - std_rs, mean_rs + std_rs, color="orange", alpha=0.2 + ) + plt.title( + f"Performance Comparison ({objective_function_choice.capitalize()} - {direction.capitalize()})" + ) + plt.xlabel("Trial Number") + plt.ylabel("Objective Value (Log Scale)") + plt.yscale("log") + plt.grid(which="both", linestyle="--", linewidth=0.5) + plt.legend() + plt.show() diff --git a/package/samplers/differential_evolution/images/Ackley_minimize.png b/package/samplers/differential_evolution/images/Ackley_minimize.png new file mode 100644 index 00000000..056f29d9 Binary files /dev/null and b/package/samplers/differential_evolution/images/Ackley_minimize.png differ diff --git a/package/samplers/differential_evolution/images/Rastrigin_minimize.png b/package/samplers/differential_evolution/images/Rastrigin_minimize.png new file mode 100644 index 00000000..c24b427e Binary files /dev/null and b/package/samplers/differential_evolution/images/Rastrigin_minimize.png differ diff --git a/package/samplers/differential_evolution/images/Schwefel_maximize.png b/package/samplers/differential_evolution/images/Schwefel_maximize.png new file mode 100644 index 00000000..964b200d Binary files /dev/null and b/package/samplers/differential_evolution/images/Schwefel_maximize.png differ diff --git a/package/samplers/differential_evolution/images/sphere_minimize.png b/package/samplers/differential_evolution/images/sphere_minimize.png new file mode 100644 index 00000000..6d443dc4 Binary files /dev/null and b/package/samplers/differential_evolution/images/sphere_minimize.png differ diff --git a/package/samplers/differential_evolution/tests/test_sampler.py b/package/samplers/differential_evolution/tests/test_sampler.py new file mode 100644 index 00000000..e6e16824 --- /dev/null +++ b/package/samplers/differential_evolution/tests/test_sampler.py @@ -0,0 +1,757 @@ +"""MIT License + +Copyright (c) 2018 Preferred Networks, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This file is taken from Optuna (https://github.com/optuna/optuna/blob/master/tests/samplers_tests/test_samplers.py) +and modified to test AutoSampler. +""" + +from __future__ import annotations + +from collections.abc import Callable +from collections.abc import Sequence +from datetime import datetime +import logging +import multiprocessing +from multiprocessing.managers import DictProxy +import os +from typing import Any +from unittest.mock import patch +import warnings + +from _pytest.fixtures import SubRequest +from _pytest.mark.structures import MarkDecorator +import numpy as np +import optuna +from optuna.distributions import BaseDistribution +from optuna.distributions import CategoricalChoiceType +from optuna.distributions import CategoricalDistribution +from optuna.distributions import FloatDistribution +from optuna.distributions import IntDistribution +from optuna.samplers import BaseSampler +from optuna.study import Study +from optuna.trial import FrozenTrial +from optuna.trial import Trial +from optuna.trial import TrialState +import optunahub +import pytest + + +# NOTE(nabenabe): This file content is mostly copied from the Optuna repository. +The_Sampler = optunahub.load_local_module( + package="package/samplers/differential_evolution", + registry_root="/home/j/experiments/optunahub-registry", +).DESampler + + +parametrize_sampler = pytest.mark.parametrize("sampler_class", [The_Sampler]) +parametrize_relative_sampler = pytest.mark.parametrize("relative_sampler_class", [The_Sampler]) +parametrize_multi_objective_sampler = pytest.mark.parametrize( + "multi_objective_sampler_class", [The_Sampler] +) + + +sampler_class_with_seed: dict[str, Callable[[int], BaseSampler]] = { + "TheSampler": lambda seed: The_Sampler(seed=seed) +} +param_sampler_with_seed = [] +param_sampler_name_with_seed = [] +for sampler_name, sampler_class in sampler_class_with_seed.items(): + param_sampler_with_seed.append(pytest.param(sampler_class, id=sampler_name)) + param_sampler_name_with_seed.append(pytest.param(sampler_name)) +parametrize_sampler_with_seed = pytest.mark.parametrize("sampler_class", param_sampler_with_seed) +parametrize_sampler_name_with_seed = pytest.mark.parametrize( + "sampler_name", param_sampler_name_with_seed +) + + +def parametrize_suggest_method(name: str) -> MarkDecorator: + return pytest.mark.parametrize( + f"suggest_method_{name}", + [ + lambda t: t.suggest_float(name, 0, 10), + lambda t: t.suggest_int(name, 0, 10), + lambda t: t.suggest_categorical(name, [0, 1, 2]), + lambda t: t.suggest_float(name, 0, 10, step=0.5), + lambda t: t.suggest_float(name, 1e-7, 10, log=True), + lambda t: t.suggest_int(name, 1, 10, log=True), + ], + ) + + +def _choose_sampler_in_auto_sampler_and_set_n_startup_trials_to_zero(study: optuna.Study) -> None: + # NOTE(nabenabe): Choose a sampler inside AutoSampler. + study.sampler.before_trial(study, trial=_create_new_trial(study)) + study.sampler._sampler._n_startup_trials = 0 + + +@parametrize_sampler +@pytest.mark.parametrize( + "distribution", + [ + FloatDistribution(-1.0, 1.0), + FloatDistribution(0.0, 1.0), + FloatDistribution(-1.0, 0.0), + FloatDistribution(1e-7, 1.0, log=True), + FloatDistribution(-10, 10, step=0.1), + FloatDistribution(-10.2, 10.2, step=0.1), + ], +) +def test_float(sampler_class: Callable[[], BaseSampler], distribution: FloatDistribution) -> None: + study = optuna.study.create_study(sampler=sampler_class()) + points = np.array( + [ + study.sampler.sample_independent(study, _create_new_trial(study), "x", distribution) + for _ in range(100) + ] + ) + assert np.all(points >= distribution.low) + assert np.all(points <= distribution.high) + assert not isinstance( + study.sampler.sample_independent(study, _create_new_trial(study), "x", distribution), + np.floating, + ) + + if distribution.step is not None: + # Check all points are multiples of distribution.step. + points -= distribution.low + points /= distribution.step + round_points = np.round(points) + np.testing.assert_almost_equal(round_points, points) + + +@parametrize_sampler +@pytest.mark.parametrize( + "distribution", + [ + IntDistribution(-10, 10), + IntDistribution(0, 10), + IntDistribution(-10, 0), + IntDistribution(-10, 10, step=2), + IntDistribution(0, 10, step=2), + IntDistribution(-10, 0, step=2), + IntDistribution(1, 100, log=True), + ], +) +def test_int(sampler_class: Callable[[], BaseSampler], distribution: IntDistribution) -> None: + study = optuna.study.create_study(sampler=sampler_class()) + points = np.array( + [ + study.sampler.sample_independent(study, _create_new_trial(study), "x", distribution) + for _ in range(100) + ] + ) + assert np.all(points >= distribution.low) + assert np.all(points <= distribution.high) + assert not isinstance( + study.sampler.sample_independent(study, _create_new_trial(study), "x", distribution), + np.integer, + ) + + +@parametrize_sampler +@pytest.mark.parametrize("choices", [(1, 2, 3), ("a", "b", "c"), (1, "a")]) +def test_categorical( + sampler_class: Callable[[], BaseSampler], choices: Sequence[CategoricalChoiceType] +) -> None: + distribution = CategoricalDistribution(choices) + + study = optuna.study.create_study(sampler=sampler_class()) + + def sample() -> float: + trial = _create_new_trial(study) + param_value = study.sampler.sample_independent(study, trial, "x", distribution) + return float(distribution.to_internal_repr(param_value)) + + points = np.asarray([sample() for i in range(100)]) + + # 'x' value is corresponding to an index of distribution.choices. + assert np.all(points >= 0) + assert np.all(points <= len(distribution.choices) - 1) + round_points = np.round(points) + np.testing.assert_almost_equal(round_points, points) + + +@parametrize_relative_sampler +@pytest.mark.parametrize( + "x_distribution", + [ + FloatDistribution(-1.0, 1.0), + FloatDistribution(1e-7, 1.0, log=True), + FloatDistribution(-10, 10, step=0.5), + IntDistribution(3, 10), + IntDistribution(1, 100, log=True), + IntDistribution(3, 9, step=2), + ], +) +@pytest.mark.parametrize( + "y_distribution", + [ + FloatDistribution(-1.0, 1.0), + FloatDistribution(1e-7, 1.0, log=True), + FloatDistribution(-10, 10, step=0.5), + IntDistribution(3, 10), + IntDistribution(1, 100, log=True), + IntDistribution(3, 9, step=2), + ], +) +def test_sample_relative_numerical( + relative_sampler_class: Callable[[], BaseSampler], + x_distribution: BaseDistribution, + y_distribution: BaseDistribution, +) -> None: + search_space: dict[str, BaseDistribution] = dict(x=x_distribution, y=y_distribution) + study = optuna.study.create_study(sampler=relative_sampler_class()) + trial = study.ask(search_space) + study.tell(trial, sum(trial.params.values())) + + """ + This test checks if the relative sampler samples parameters correctly. Specifically, it checks + if the sampled parameters are within the search space and have the correct type. The test + samples 10 points and checks if the sampled parameters are within the search space and have the + correct type. + """ + + def sample() -> list[int | float]: + params = study.sampler.sample_relative(study, _create_new_trial(study), search_space) + return [params[name] for name in search_space] + + points = np.array([sample() for _ in range(10)]) + for i, distribution in enumerate(search_space.values()): + assert isinstance( + distribution, + ( + FloatDistribution, + IntDistribution, + ), + ), "The distribution must be either FloatDistribution or IntDistribution." + assert np.all(points[:, i] >= distribution.low), "The sampled value must be >= low." + assert np.all(points[:, i] <= distribution.high), "The sampled value must be <= high." + for param_value, distribution in zip(sample(), search_space.values()): + assert not isinstance( + param_value, np.floating + ), f"The sampled value must not be a numpy float, instead it is {param_value}" + assert not isinstance( + param_value, np.integer + ), f"The sampled value must not be a numpy integer, instead it is {param_value}" + if isinstance(distribution, IntDistribution): + assert isinstance(param_value, int), "The sampled value must be an integer." + else: + assert isinstance(param_value, float), "The sampled value must be a float." + + +@parametrize_relative_sampler +def test_sample_relative_categorical(relative_sampler_class: Callable[[], BaseSampler]) -> None: + search_space: dict[str, BaseDistribution] = dict( + x=CategoricalDistribution([1, 10, 100]), y=CategoricalDistribution([-1, -10, -100]) + ) + study = optuna.study.create_study(sampler=relative_sampler_class()) + trial = study.ask(search_space) + study.tell(trial, sum(trial.params.values())) + + def sample() -> list[float]: + params = study.sampler.sample_relative(study, _create_new_trial(study), search_space) + return [params[name] for name in search_space] + + points = np.array([sample() for _ in range(10)]) + for i, distribution in enumerate(search_space.values()): + assert isinstance(distribution, CategoricalDistribution) + assert np.all([v in distribution.choices for v in points[:, i]]) + for param_value in sample(): + assert not isinstance(param_value, np.floating) + assert not isinstance(param_value, np.integer) + assert isinstance(param_value, int) + + +@parametrize_relative_sampler +@pytest.mark.parametrize( + "x_distribution", + [ + FloatDistribution(-1.0, 1.0), + FloatDistribution(1e-7, 1.0, log=True), + FloatDistribution(-10, 10, step=0.5), + IntDistribution(1, 10), + IntDistribution(1, 100, log=True), + ], +) +def test_sample_relative_mixed( + relative_sampler_class: Callable[[], BaseSampler], x_distribution: BaseDistribution +) -> None: + search_space: dict[str, BaseDistribution] = dict( + x=x_distribution, y=CategoricalDistribution([-1, -10, -100]) + ) + study = optuna.study.create_study(sampler=relative_sampler_class()) + trial = study.ask(search_space) + study.tell(trial, sum(trial.params.values())) + + def sample() -> list[float]: + params = study.sampler.sample_relative(study, _create_new_trial(study), search_space) + return [params[name] for name in search_space] + + points = np.array([sample() for _ in range(10)]) + assert isinstance( + search_space["x"], + ( + FloatDistribution, + IntDistribution, + ), + ) + assert np.all(points[:, 0] >= search_space["x"].low) + assert np.all(points[:, 0] <= search_space["x"].high) + assert isinstance(search_space["y"], CategoricalDistribution) + assert np.all([v in search_space["y"].choices for v in points[:, 1]]) + for param_value, distribution in zip(sample(), search_space.values()): + assert not isinstance(param_value, np.floating) + assert not isinstance(param_value, np.integer) + if isinstance( + distribution, + ( + IntDistribution, + CategoricalDistribution, + ), + ): + assert isinstance(param_value, int) + else: + assert isinstance(param_value, float) + + +@parametrize_sampler +def test_conditional_sample_independent(sampler_class: Callable[[], BaseSampler]) -> None: + # This test case reproduces the error reported in #2734. + # See https://github.com/optuna/optuna/pull/2734#issuecomment-857649769. + + study = optuna.study.create_study(sampler=sampler_class()) + categorical_distribution = CategoricalDistribution(choices=["x", "y"]) + dependent_distribution = CategoricalDistribution(choices=["a", "b"]) + + study.add_trial( + optuna.create_trial( + params={"category": "x", "x": "a"}, + distributions={"category": categorical_distribution, "x": dependent_distribution}, + value=0.1, + ) + ) + + study.add_trial( + optuna.create_trial( + params={"category": "y", "y": "b"}, + distributions={"category": categorical_distribution, "y": dependent_distribution}, + value=0.1, + ) + ) + + _trial = _create_new_trial(study) + category = study.sampler.sample_independent( + study, _trial, "category", categorical_distribution + ) + assert category in ["x", "y"] + value = study.sampler.sample_independent(study, _trial, category, dependent_distribution) + assert value in ["a", "b"] + + +def _create_new_trial(study: Study) -> FrozenTrial: + trial_id = study._storage.create_new_trial(study._study_id) + return study._storage.get_trial(trial_id) + + +@parametrize_sampler +def test_nan_objective_value(sampler_class: Callable[[], BaseSampler]) -> None: + study = optuna.create_study(sampler=sampler_class()) + + def objective(trial: Trial, base_value: float) -> float: + return trial.suggest_float("x", 0.1, 0.2) + base_value + + # Non NaN objective values. + for i in range(10, 1, -1): + study.optimize(lambda t: objective(t, i), n_trials=1, catch=()) + assert int(study.best_value) == 2 + + # NaN objective values. + study.optimize(lambda t: objective(t, float("nan")), n_trials=1, catch=()) + assert int(study.best_value) == 2 + + # Non NaN objective value. + study.optimize(lambda t: objective(t, 1), n_trials=1, catch=()) + assert int(study.best_value) == 1 + + +@parametrize_multi_objective_sampler +@pytest.mark.parametrize( + "distribution", + [ + FloatDistribution(-1.0, 1.0), + FloatDistribution(0.0, 1.0), + FloatDistribution(-1.0, 0.0), + FloatDistribution(1e-7, 1.0, log=True), + FloatDistribution(-10, 10, step=0.1), + FloatDistribution(-10.2, 10.2, step=0.1), + IntDistribution(-10, 10), + IntDistribution(0, 10), + IntDistribution(-10, 0), + IntDistribution(-10, 10, step=2), + IntDistribution(0, 10, step=2), + IntDistribution(-10, 0, step=2), + IntDistribution(1, 100, log=True), + CategoricalDistribution((1, 2, 3)), + CategoricalDistribution(("a", "b", "c")), + CategoricalDistribution((1, "a")), + ], +) +def test_multi_objective_sample_independent( + multi_objective_sampler_class: Callable[[], BaseSampler], distribution: BaseDistribution +) -> None: + study = optuna.study.create_study( + directions=["minimize", "maximize"], sampler=multi_objective_sampler_class() + ) + for i in range(100): + value = study.sampler.sample_independent( + study, _create_new_trial(study), "x", distribution + ) + assert distribution._contains(distribution.to_internal_repr(value)) + + if not isinstance(distribution, CategoricalDistribution): + # Please see https://github.com/optuna/optuna/pull/393 why this assertion is needed. + assert not isinstance(value, np.floating) + + if isinstance(distribution, FloatDistribution): + if distribution.step is not None: + # Check the value is a multiple of `distribution.step` which is + # the quantization interval of the distribution. + value -= distribution.low + value /= distribution.step + round_value = np.round(value) + np.testing.assert_almost_equal(round_value, value) + + +@parametrize_sampler +def test_sample_single_distribution(sampler_class: Callable[[], BaseSampler]) -> None: + relative_search_space = { + "a": CategoricalDistribution([1]), + "b": IntDistribution(low=1, high=1), + "c": IntDistribution(low=1, high=1, log=True), + "d": FloatDistribution(low=1.0, high=1.0), + "e": FloatDistribution(low=1.0, high=1.0, log=True), + "f": FloatDistribution(low=1.0, high=1.0, step=1.0), + } + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning) + sampler = sampler_class() + study = optuna.study.create_study(sampler=sampler) + + # We need to test the construction of the model, so we should set `n_trials >= 2`. + for _ in range(2): + trial = study.ask(fixed_distributions=relative_search_space) + study.tell(trial, 1.0) + for param_name in relative_search_space.keys(): + assert trial.params[param_name] == 1 + + +@parametrize_sampler +@parametrize_suggest_method("x") +def test_single_parameter_objective( + sampler_class: Callable[[], BaseSampler], suggest_method_x: Callable[[Trial], float] +) -> None: + def objective(trial: Trial) -> float: + return suggest_method_x(trial) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning) + sampler = sampler_class() + + study = optuna.study.create_study(sampler=sampler) + study.optimize(objective, n_trials=10) + + assert len(study.trials) == 10 + assert all(t.state == TrialState.COMPLETE for t in study.trials) + + +@parametrize_sampler +def test_conditional_parameter_objective(sampler_class: Callable[[], BaseSampler]) -> None: + def objective(trial: Trial) -> float: + x = trial.suggest_categorical("x", [True, False]) + if x: + return trial.suggest_float("y", 0, 1) + return trial.suggest_float("z", 0, 1) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning) + sampler = sampler_class() + + study = optuna.study.create_study(sampler=sampler) + study.optimize(objective, n_trials=10) + + assert len(study.trials) == 10 + assert all(t.state == TrialState.COMPLETE for t in study.trials) + + +@parametrize_sampler +@parametrize_suggest_method("x") +@parametrize_suggest_method("y") +def test_combination_of_different_distributions_objective( + sampler_class: Callable[[], BaseSampler], + suggest_method_x: Callable[[Trial], float], + suggest_method_y: Callable[[Trial], float], +) -> None: + def objective(trial: Trial) -> float: + return suggest_method_x(trial) + suggest_method_y(trial) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning) + sampler = sampler_class() + + study = optuna.study.create_study(sampler=sampler) + study.optimize(objective, n_trials=3) + + assert len(study.trials) == 3 + assert all(t.state == TrialState.COMPLETE for t in study.trials) + + +@parametrize_sampler +@pytest.mark.parametrize( + "second_low,second_high", + [ + (0, 5), # Narrow range. + (0, 20), # Expand range. + (20, 30), # Set non-overlapping range. + ], +) +def test_dynamic_range_objective( + sampler_class: Callable[[], BaseSampler], second_low: int, second_high: int +) -> None: + def objective(trial: Trial, low: int, high: int) -> float: + v = trial.suggest_float("x", low, high) + v += trial.suggest_int("y", low, high) + return v + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning) + sampler = sampler_class() + + study = optuna.study.create_study(sampler=sampler) + study.optimize(lambda t: objective(t, 0, 10), n_trials=10) + study.optimize(lambda t: objective(t, second_low, second_high), n_trials=10) + + assert len(study.trials) == 20 + assert all(t.state == TrialState.COMPLETE for t in study.trials) + + +# We add tests for constant objective functions to ensure the reproducibility of sorting. +@parametrize_sampler_with_seed +@pytest.mark.slow +@pytest.mark.parametrize("objective_func", [lambda *args: sum(args), lambda *args: 0.0]) +def test_reproducible(sampler_class: Callable[[int], BaseSampler], objective_func: Any) -> None: + def objective(trial: Trial) -> float: + a = trial.suggest_float("a", 1, 9) + b = trial.suggest_float("b", 1, 9, log=True) + c = trial.suggest_float("c", 1, 9, step=1) + d = trial.suggest_int("d", 1, 9) + e = trial.suggest_int("e", 1, 9, log=True) + f = trial.suggest_int("f", 1, 9, step=2) + g = trial.suggest_categorical("g", range(1, 10)) + return objective_func(a, b, c, d, e, f, g) + + """ + This test checks if the sampler is reproducible. Specifically, it checks if the same set of + parameters are suggested for the same seed. The test optimizes a constant objective function + with 15 trials and checks if the parameters are the same for the same seed. + """ + + study = optuna.create_study(sampler=sampler_class(1)) + study.optimize(objective, n_trials=15) + + study_same_seed = optuna.create_study(sampler=sampler_class(1)) + study_same_seed.optimize(objective, n_trials=15) + for i in range(15): + assert study.trials[i].params == study_same_seed.trials[i].params + + study_different_seed = optuna.create_study(sampler=sampler_class(2)) + study_different_seed.optimize(objective, n_trials=15) + assert any( + [study.trials[i].params != study_different_seed.trials[i].params for i in range(15)] + ) + + +@pytest.mark.slow +@parametrize_sampler_with_seed +def test_reseed_rng_change_sampling(sampler_class: Callable[[int], BaseSampler]) -> None: + def objective(trial: Trial) -> float: + a = trial.suggest_float("a", 1, 9) + b = trial.suggest_float("b", 1, 9, log=True) + c = trial.suggest_float("c", 1, 9, step=1) + d = trial.suggest_int("d", 1, 9) + e = trial.suggest_int("e", 1, 9, log=True) + f = trial.suggest_int("f", 1, 9, step=2) + g = trial.suggest_categorical("g", range(1, 10)) + return a + b + c + d + e + f + g + + sampler = sampler_class(1) + study = optuna.create_study(sampler=sampler) + study.optimize(objective, n_trials=15) + + sampler_different_seed = sampler_class(1) + sampler_different_seed.reseed_rng() + study_different_seed = optuna.create_study(sampler=sampler_different_seed) + study_different_seed.optimize(objective, n_trials=15) + assert any( + [study.trials[i].params != study_different_seed.trials[i].params for i in range(15)] + ) + + +# This function is used only in test_reproducible_in_other_process, but declared at top-level +# because local function cannot be pickled, which occurs within multiprocessing. +def run_optimize( + k: int, + sampler_name: str, + sequence_dict: DictProxy, + hash_dict: DictProxy, + log_file: str, # change from Jinglue: added log_file parameter +) -> None: + try: # change from Jinglue: added try-except block for error handling + # change from Jinglue: added logging setup + logging.basicConfig( + filename=log_file, + level=logging.DEBUG, + format=f"Process {k} - %(asctime)s - %(message)s", + ) + + logging.info(f"Starting process {k}") # change from Jinglue: added logging + hash_dict[k] = hash("nondeterministic hash") + logging.info("Hash stored") # change from Jinglue: added logging + + def objective(trial: Trial) -> float: + a = trial.suggest_float("a", 1, 9) + b = trial.suggest_float("b", 1, 9, log=True) + c = trial.suggest_float("c", 1, 9, step=1) + d = trial.suggest_int("d", 1, 9) + e = trial.suggest_int("e", 1, 9, log=True) + f = trial.suggest_int("f", 1, 9, step=2) + g = trial.suggest_categorical("g", range(1, 10)) + return a + b + c + d + e + f + g + + logging.info("Creating sampler") # change from Jinglue: added logging + sampler = sampler_class_with_seed[sampler_name](1) + logging.info("Creating study") # change from Jinglue: added logging + study = optuna.create_study(sampler=sampler) + logging.info("Starting optimization") # change from Jinglue: added logging + study.optimize(objective, n_trials=15) + sequence_dict[k] = list(study.trials[-1].params.values()) + logging.info("Optimization complete") # change from Jinglue: added logging + + except Exception as e: # change from Jinglue: added error handling + import traceback + + logging.error(f"Error occurred: {str(e)}") + logging.error(traceback.format_exc()) + raise + + +@pytest.fixture +def unset_seed_in_test(request: SubRequest) -> None: + # Unset the hashseed at beginning and restore it at end regardless of an exception in the test. + # See https://docs.pytest.org/en/stable/how-to/fixtures.html#adding-finalizers-directly + # for details. + + hash_seed = os.getenv("PYTHONHASHSEED") + if hash_seed is not None: + del os.environ["PYTHONHASHSEED"] + + def restore_seed() -> None: + if hash_seed is not None: + os.environ["PYTHONHASHSEED"] = hash_seed + + request.addfinalizer(restore_seed) + + +@pytest.mark.slow +@parametrize_sampler_name_with_seed +def test_reproducible_in_other_process(sampler_name: str, unset_seed_in_test: None) -> None: + multiprocessing.set_start_method("spawn", force=True) + + # change from Jinglue: added log file creation + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = f"test_sampler_{timestamp}.log" + + manager = multiprocessing.Manager() + sequence_dict: DictProxy = manager.dict() + hash_dict: DictProxy = manager.dict() + + # change from Jinglue: modified process management to track all processes + processes = [] + for i in range(3): + p = multiprocessing.Process( + target=run_optimize, args=(i, sampler_name, sequence_dict, hash_dict, log_file) + ) + processes.append(p) + p.start() + + # change from Jinglue: modified process checking and added error reporting + failed = False + for i, p in enumerate(processes): + p.join() + if p.exitcode != 0: + failed = True + + # change from Jinglue: added log file checking on failure + if failed: + print("\nTest failed! Log contents:") + print("-" * 50) + try: + with open(log_file, "r") as f: + print(f.read()) + except Exception as e: + print(f"Error reading log file: {str(e)}") + print("-" * 50) + os.remove(log_file) # Clean up log file + raise RuntimeError("One or more processes failed. See log contents above.") + + os.remove(log_file) # change from Jinglue: added log file cleanup + + # Rest of the assertions... + assert not ( + hash_dict[0] == hash_dict[1] == hash_dict[2] + ), "Hashes are expected to be different" + assert ( + sequence_dict[0] == sequence_dict[1] == sequence_dict[2] + ), "Sequences are expected to be same" + + +@pytest.mark.parametrize("n_jobs", [1, 2]) +@parametrize_relative_sampler +def test_cache_is_invalidated( + n_jobs: int, relative_sampler_class: Callable[[], BaseSampler] +) -> None: + sampler = relative_sampler_class() + original_before_trial = sampler.before_trial + + def mock_before_trial(study: Study, trial: FrozenTrial) -> None: + assert study._thread_local.cached_all_trials is None + original_before_trial(study, trial) + + with patch.object(sampler, "before_trial", side_effect=mock_before_trial): + study = optuna.study.create_study(sampler=sampler) + + def objective(trial: Trial) -> float: + assert trial._relative_params is None + + trial.suggest_float("x", -10, 10) + trial.suggest_float("y", -10, 10) + assert trial._relative_params is not None + return -1 + + study.optimize(objective, n_trials=10, n_jobs=n_jobs)