diff --git a/package/benchmarks/bbob/LICENSE b/package/benchmarks/bbob/LICENSE new file mode 100644 index 0000000..380b0c2 --- /dev/null +++ b/package/benchmarks/bbob/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Preferred Networks, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/package/benchmarks/bbob/README.md b/package/benchmarks/bbob/README.md new file mode 100644 index 0000000..6c89252 --- /dev/null +++ b/package/benchmarks/bbob/README.md @@ -0,0 +1,106 @@ +--- +author: Optuna team +title: The blackbox optimization benchmarking (bbob) test suite +description: The blackbox optimization benchmarking (bbob) test suite consists of 24 noiseless single-objective test functions including Sphere, Ellipsoidal, Rastrigin, Rosenbrock, etc. This package is a wrapper of the COCO (COmparing Continuous Optimizers) experiments library. +tags: [benchmark, continuous optimization, BBOB, COCO] +optuna_versions: [4.1.0] +license: MIT License +--- + +## Abstract + +The blackbox optimization benchmarking (bbob) test suite comprises 24 noiseless single-objective test functions. BBOB is one of the most widely used test suites to evaluate and compare the performance of blackbox optimization algorithms. Each benchmark function is provided in dimensions \[2, 3, 5, 10, 20, 40\] with 110 instances. + +## APIs + +### class `Problem(function_id: int, dimension: int, instance_id: int = 1)` + +- `function_id`: [ID of the bbob benchmark function](https://numbbo.github.io/coco/testsuites/bbob) to use. It must be in the range of `[1, 24]`. +- `dimension`: Dimension of the benchmark function. It must be in `[2, 3, 5, 10, 20, 40]`. +- `instance_id`: ID of the instance of the benchmark function. It must be in the range of `[1, 110]`. + +#### Methods and Properties + +- `search_space`: Return the search space. + - Returns: `dict[str, optuna.distributions.BaseDistribution]` +- `directions`: Return the optimization directions. + - Returns: `list[optuna.study.StudyDirection]` +- `__call__(trial: optuna.Trial)`: Evaluate the objective function and return the objective value. + - Args: + - `trial`: Optuna trial object. + - Returns: `float` +- `evaluate(params: dict[str, float])`: Evaluate the objective function given a dictionary of parameters. + - Args: + - `params`: Decision variable like `{"x0": x1_value, "x1": x1_value, ..., "xn": xn_value}`. The number of parameters must be equal to `dimension`. + - Returns: `float` + +The properties defined by [cocoex.Problem](https://numbbo.github.io/coco-doc/apidocs/cocoex/cocoex.Problem.html) are also available such as `number_of_objectives`. + +## Installation + +Please install the [coco-experiment](https://github.com/numbbo/coco-experiment/tree/main/build/python) package. + +```shell +pip install -U coco-experiment +``` + +## Example + +```python +import optuna +import optunahub + + +bbob = optunahub.load_module("benchmarks/bbob") +sphere2d = bbob.Problem(function_id=1, dimension=2, instance_id=1) + +study = optuna.create_study(directions=sphere2d.directions) +study.optimize(sphere2d, n_trials=20) + +print(study.best_trial.params, study.best_trial.value) +``` + +## List of Benchmark Functions + +Please refer to [the paper](https://numbbo.github.io/gforge/downloads/download16.00/bbobdocfunctions.pdf) for details about each benchmark function. + +**Category** + +1. Separable Functions +1. Functions with low or moderate conditioning +1. Functions with high conditioning and unimodal +1. Multi-modal functions with adequate global structure +1. Multi-modal functions with weak global structure + +| Category | Function ID | Function Name | +|-----------|-------------|--------------------------------------------------------------------------------------------------------------------------| +| 1 | 1 | [Sphere Function](https://coco-platform.org/testsuites/bbob/functions/f01.html) | +| 1 | 2 | [Separable Ellipsoidal Function](https://coco-platform.org/testsuites/bbob/functions/f02.html) | +| 1 | 3 | [Rastrigin Function](https://coco-platform.org/testsuites/bbob/functions/f03.html) | +| 1 | 4 | [Büche-Rastrigin Function](https://coco-platform.org/testsuites/bbob/functions/f04.html) | +| 1 | 5 | [Linear Slope](https://coco-platform.org/testsuites/bbob/functions/f05.html) | +| 2 | 6 | [Attractive Sector Function](https://coco-platform.org/testsuites/bbob/functions/f06.html) | +| 2 | 7 | [Step Ellipsoidal Function](https://coco-platform.org/testsuites/bbob/functions/f07.html) | +| 2 | 8 | [Rosenbrock Function, original](https://coco-platform.org/testsuites/bbob/functions/f08.html) | +| 2 | 9 | [Rosenbrock Function, rotated](https://coco-platform.org/testsuites/bbob/functions/f09.html) | +| 3 | 10 | [Ellipsoidal Function](https://coco-platform.org/testsuites/bbob/functions/f10.html) | +| 3 | 11 | [Discus Function](https://coco-platform.org/testsuites/bbob/functions/f11.html) | +| 3 | 12 | [Bent Cigar Function](https://coco-platform.org/testsuites/bbob/functions/f12.html) | +| 3 | 13 | [Sharp Ridge Function](https://coco-platform.org/testsuites/bbob/functions/f13.html) | +| 3 | 14 | [Different Powers Function](https://coco-platform.org/testsuites/bbob/functions/f14.html) | +| 4 | 15 | [Rastrigin Function](https://coco-platform.org/testsuites/bbob/functions/f15.html) | +| 4 | 16 | [Weierstrass Function](https://coco-platform.org/testsuites/bbob/functions/f16.html) | +| 4 | 17 | [Schaffer's F7 Function](https://coco-platform.org/testsuites/bbob/functions/f17.html) | +| 4 | 18 | [Schaffer's F7 Function, moderately ill-conditioned](https://coco-platform.org/testsuites/bbob/functions/f18.html) | +| 4 | 19 | [Composite Griewank-Rosenbrock Function F8F2](https://coco-platform.org/testsuites/bbob/functions/f19.html) | +| 5 | 20 | [Schwefel Function](https://coco-platform.org/testsuites/bbob/functions/f20.html) | +| 5 | 21 | [Gallagher's Gaussian 101-me Peaks Function](https://coco-platform.org/testsuites/bbob/functions/f21.html) | +| 5 | 22 | [Gallagher's Gaussian 21-hi Peaks Function](https://coco-platform.org/testsuites/bbob/functions/f22.html) | +| 5 | 23 | [Katsuura Function](https://coco-platform.org/testsuites/bbob/functions/f23.html) | +| 5 | 24 | [Lunacek bi-Rastrigin Function](https://coco-platform.org/testsuites/bbob/functions/f24.html) | + +![BBOB Plots](images/bbob.png) + +## Reference + +Finck, S., Hansen, N., Ros, R., & Auger, A. [Real-Parameter Black-Box Optimization Benchmarking 2010: Presentation of the Noiseless Functions](https://numbbo.github.io/gforge/downloads/download16.00/bbobdocfunctions.pdf). diff --git a/package/benchmarks/bbob/__init__.py b/package/benchmarks/bbob/__init__.py new file mode 100644 index 0000000..2a6de17 --- /dev/null +++ b/package/benchmarks/bbob/__init__.py @@ -0,0 +1,4 @@ +from ._bbob import Problem + + +__all__ = ["Problem"] diff --git a/package/benchmarks/bbob/_bbob.py b/package/benchmarks/bbob/_bbob.py new file mode 100644 index 0000000..d69227b --- /dev/null +++ b/package/benchmarks/bbob/_bbob.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from typing import Any + +import cocoex as ex +import optuna +import optunahub + + +class Problem(optunahub.benchmarks.BaseProblem): + """Wrapper class for COCO bbob test suite. + https://coco-platform.org/testsuites/bbob/overview.html + + 1 Separable Functions + f1: Sphere Function + f2: Separable Ellipsoidal Function + f3: Rastrigin Function + f4: Büche-Rastrigin Function + f5: Linear Slope + 2 Functions with low or moderate conditioning + f6: Attractive Sector Function + f7: Step Ellipsoidal Function + f8: Rosenbrock Function, original + f9: Rosenbrock Function, rotated + 3 Functions with high conditioning and unimodal + f10: Ellipsoidal Function + f11: Discus Function + f12: Bent Cigar Function + f13: Sharp Ridge Function + f14: Different Powers Function + 4 Multi-modal functions with adequate global structure + f15: Rastrigin Function + f16: Weierstrass Function + f17: Schaffer's F7 Function + f18: Schaffer's F7 Function, moderately ill-conditioned + f19: Composite Griewank-Rosenbrock Function F8F2 + 5 Multi-modal functions with weak global structure + f20: Schwefel Function + f21: Gallagher's Gaussian 101-me Peaks Function + f22: Gallagher's Gaussian 21-hi Peaks Function + f23: Katsuura Function + f24: Lunacek bi-Rastrigin Function + """ + + def __init__(self, function_id: int, dimension: int, instance_id: int = 1): + """Initialize the problem. + Args: + function_id: Function index in [1, 24]. + dimension: Dimension of the problem in [2, 3, 5, 10, 20, 40]. + instance_id: Instance index in [1, 110]. + + Please refer to the COCO documentation for the details of the available properties. + https://numbbo.github.io/coco-doc/apidocs/cocoex/cocoex.Problem.html + """ + + assert 1 <= function_id <= 24, "function_id must be in [1, 24]" + assert dimension in [2, 3, 5, 10, 20, 40], "dimension must be in [2, 3, 5, 10, 20, 40]" + assert 1 <= instance_id <= 110, "instance_id must be in [1, 110]" + + self._problem = ex.Suite("bbob", "", "").get_problem_by_function_dimension_instance( + function=function_id, dimension=dimension, instance=instance_id + ) + self._search_space = { + f"x{i}": optuna.distributions.FloatDistribution( + low=self._problem.lower_bounds[i], + high=self._problem.upper_bounds[i], + ) + for i in range(self._problem.dimension) + } + + @property + def search_space(self) -> dict[str, optuna.distributions.BaseDistribution]: + """Return the search space.""" + return self._search_space.copy() + + @property + def directions(self) -> list[optuna.study.StudyDirection]: + """Return the optimization directions.""" + return [optuna.study.StudyDirection.MINIMIZE] + + def evaluate(self, params: dict[str, float]) -> float: + """Evaluate the objective function. + Args: + params: + Decision variable, e.g., evaluate({"x0": 1.0, "x1": 2.0}). + The number of parameters must be equal to the dimension of the problem. + Returns: + The objective value. + + """ + return self._problem([params[name] for name in self._search_space]) + + def __getattr__(self, name: str) -> Any: + return getattr(self._problem, name) + + def __del__(self) -> None: + self._problem.free() diff --git a/package/benchmarks/bbob/images/bbob.png b/package/benchmarks/bbob/images/bbob.png new file mode 100644 index 0000000..b3a6ca0 Binary files /dev/null and b/package/benchmarks/bbob/images/bbob.png differ diff --git a/package/benchmarks/bbob/requirements.txt b/package/benchmarks/bbob/requirements.txt new file mode 100644 index 0000000..8213f25 --- /dev/null +++ b/package/benchmarks/bbob/requirements.txt @@ -0,0 +1 @@ +coco-experiment \ No newline at end of file