Made the hill climb algorithm and tested locally

package/samplers/hill_climb_search/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Chinmaya Sahu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/samplers/hill_climb_search/README.md

@@ -0,0 +1,110 @@
+---
+author: Please fill in the author name here. (e.g., John Smith)
+title: Please fill in the title of the feature here. (e.g., Gaussian-Process Expected Improvement Sampler)
+description: Please fill in the description of the feature here. (e.g., This sampler searches for each trial based on expected improvement using Gaussian process.)
+tags: [Please fill in the list of tags here. (e.g., sampler, visualization, pruner)]
+optuna_versions: ['Please fill in the list of versions of Optuna in which you have confirmed the feature works, e.g., 3.6.1.']
+license: MIT License
+---
+
+<!--
+This is an example of the frontmatters.
+All columns must be string.
+You can omit quotes when value types are not ambiguous.
+For tags, a package placed in
+- package/samplers/ must include the tag "sampler"
+- package/visualilzation/ must include the tag "visualization"
+- package/pruners/ must include the tag "pruner"
+respectively.
+
+---
+author: Optuna team
+title: My Sampler
+description: A description for My Sampler.
+tags: [sampler, 2nd tag for My Sampler, 3rd tag for My Sampler]
+optuna_versions: [3.6.1]
+license: "MIT License"
+---
+-->
+
+Please read the [tutorial guide](https://optuna.github.io/optunahub-registry/recipes/001_first.html) to register your feature in OptunaHub.
+You can find more detailed explanation of the following contents in the tutorial.
+Looking at [other packages' implementations](https://github.com/optuna/optunahub-registry/tree/main/package) will also help you.
+
+## Abstract
+
+You can provide an abstract for your package here.
+This section will help attract potential users to your package.
+
+**Example**
+
+This package provides a sampler based on Gaussian process-based Bayesian optimization. The sampler is highly sample-efficient, so it is suitable for computationally expensive optimization problems with a limited evaluation budget, such as hyperparameter optimization of machine learning algorithms.
+
+## Class or Function Names
+
+Please fill in the class/function names which you implement here.
+
+**Example**
+
+- GPSampler
+
+## Installation
+
+If you have additional dependencies, please fill in the installation guide here.
+If no additional dependencies is required, **this section can be removed**.
+
+**Example**
+
+```shell
+$ pip install scipy torch
+```
+
+If your package has `requirements.txt`, it will be automatically uploaded to the OptunaHub, and the package dependencies will be available to install as follows.
+
+```shell
+ pip install -r https://hub.optuna.org/{category}/{your_package_name}/requirements.txt
+```
+
+## Example
+
+Please fill in the code snippet to use the implemented feature here.
+
+**Example**
+
+```python
+import optuna
+import optunahub
+
+
+def objective(trial):
+  x = trial.suggest_float("x", -5, 5)
+  return x**2
+
+
+sampler = optunahub.load_module(package="samplers/gp").GPSampler()
+study = optuna.create_study(sampler=sampler)
+study.optimize(objective, n_trials=100)
+```
+
+## Others
+
+Please fill in any other information if you have here by adding child sections (###).
+If there is no additional information, **this section can be removed**.
+
+<!--
+For example, you can add sections to introduce a corresponding paper.
+
+### Reference
+Takuya Akiba, Shotaro Sano, Toshihiko Yanase, Takeru Ohta, and Masanori Koyama. 2019.
+Optuna: A Next-generation Hyperparameter Optimization Framework. In KDD.
+
+### Bibtex
+```
+@inproceedings{optuna_2019,
+    title={Optuna: A Next-generation Hyperparameter Optimization Framework},
+    author={Akiba, Takuya and Sano, Shotaro and Yanase, Toshihiko and Ohta, Takeru and Koyama, Masanori},
+    booktitle={Proceedings of the 25th {ACM} {SIGKDD} International Conference on Knowledge Discovery and Data Mining},
+    year={2019}
+}
+```
+-->

package/samplers/hill_climb_search/__init__.py

@@ -0,0 +1,4 @@
+from .hill_climb_search import HillClimbSearch
+
+
+__all__ = ["HillClimbSearch"]

package/samplers/hill_climb_search/example.py

@@ -0,0 +1,16 @@
+import optuna
+import optunahub
+
+if __name__ == "__main__":
+    mod = optunahub.load_module("samplers/hill_climb_search")
+
+    def objective(trial):
+        x = trial.suggest_discrete_uniform("x", -10, 10)
+        y = trial.suggest_discrete_uniform("y", -10, 10)
+        return -(x**2 + y**2)
+
+    sampler = mod.HillClimbSearch()
+    study = optuna.create_study(sampler=sampler)
+    study.optimize(objective, n_trials=20)
+
+    print(study.best_trial.value, study.best_trial.params)

package/samplers/hill_climb_search/hill_climb_search.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import optuna
+import optunahub
+
+class HillClimbSearch(optunahub.samplers.SimpleBaseSampler):
+    """A sampler based on the Hill Climb Local Search Algorithm dealing with discrete values.
+    
+        Args:
+            
+
+    """
+
+    def __init__(self,search_space: dict[str, optuna.distributions.BaseDistribution] | None = None) -> None:
+        super().__init__(search_space)
+        self._remaining_points = []
+        self._rng = np.random.RandomState()
+        self._current_point = None
+        self._current_point_value = None
+        self._current_state = "Not Initialized"
+        self._best_neighbor = None
+        self._best_neighbor_value = None
+
+    def _generate_random_point(self, search_space):
+        params = {}
+        for param_name, param_distribution in search_space.items():
+            if isinstance(param_distribution, optuna.distributions.FloatDistribution):
+                total_points = int((param_distribution.high - param_distribution.low) / param_distribution.step)
+                params[param_name] = param_distribution.low + self._rng.randint(0, total_points)*param_distribution.step
+            else:
+                raise NotImplementedError
+        return params
+
+    def _remove_tried_points(self, neighbors, search_space, current_point):
+        final_neighbors = []
+
+        tried_points = [trial.params for trial in study.get_trials(deepcopy=False)]
+        points_to_try = self._remaining_points
+
+        invalid_points = tried_points + points_to_try + [current_point]
+
+        for neighbor in neighbors:
+            if neighbor not in invalid_points:
+                final_neighbors.append(neighbor)
+
+        return final_neighbors        
+
+    def _generate_neighbors(self, current_point, search_space):
+        neighbors = []
+        for param_name, param_distribution in search_space.items():
+            if isinstance(param_distribution, optuna.distributions.FloatDistribution):
+                current_value = current_point[param_name]
+                step = param_distribution.step
+
+                neighbor_low = max(param_distribution.low, current_value - step)
+                neighbor_high = min(param_distribution.high, current_value + step)
+
+                neighbor_low_point = current_point.copy()
+                neighbor_low_point[param_name] = neighbor_low
+                neighbor_high_point = current_point.copy()
+                neighbor_high_point[param_name] = neighbor_high
+
+                neighbors.append(neighbor_low_point)
+                neighbors.append(neighbor_high_point)
+            else:
+                raise NotImplementedError
+
+        valid_neighbors = self._remove_tried_points(neighbors, search_space, current_point)        
+
+        return valid_neighbors
+
+    def _get_previous_trial_value(self, study:optuna.study.Study) -> float:
+        if len(study.trials) > 1:
+            return study.trials[-2].value
+        else:
+            return None
+
+    def sample_relative(self, study:optuna.study.Study, trial:optuna.trial.FrozenTrial, search_space: dict[str, optuna.distributions.BaseDistribution]) -> dict[str, Any]:
+        if search_space == {}:
+            return {}
+
+        if self._current_state == "Not Initialized":
+            #Create the current point
+            starting_point = self._generate_random_point(search_space)  
+            self._current_point = starting_point
+
+            #Add the neighbours
+            neighbors = self._generate_neighbors(starting_point, search_space)
+            self._remaining_points.extend(neighbors)
+
+            #Change the state to initialized
+            self._current_state = "Initialized"
+
+            #Return the current point
+            return starting_point
+
+        elif self._current_state == "Initialized":
+            #This section is only for storing the value of the current point and best neighbor point
+            previous_trial = study.get_trials(deepcopy=False)[-2]
+            if previous_trial.params == self._current_point:
+                #Just now the current point was evaluated
+                #Store the value of the current point
+                self._current_point_value = previous_trial.value
+            else:
+                #The neighbour was evaluated
+                #Store the value of the neighbour, if it improves upon the current point
+                neighbor_value = previous_trial.value
+
+                if neighbor_value < self._current_point_value:
+                    self._best_neighbor = previous_trial.params
+                    self._best_neighbor_value = neighbor_value
+
+            #This section is for the next point to be evaluated
+            if len(self._remaining_points) == 0:
+                #This means that all the neighbours have been processed
+                #Now you have to select the best neighbour
+                #Change the state to Neighbours Processed
+                self._current_state = "Neighbours Processed"
+
+                if self._best_neighbor is not None:
+                    #Select the best neighbour, make that the current point and add its neighbours
+                    self._current_point = self._best_neighbor
+                    self._current_point_value = self._best_neighbor_value
+
+                    self._best_neighbor = None
+                    self._best_neighbor_value = None
+                    self._remaining_points = [] #Just for clarity
+
+                    #Add the neighbours
+                    neighbors = self._generate_neighbors(self._current_point, search_space)
+                    self._remaining_points.extend(neighbors)
+
+                    self._current_state = "Initialized"
+
+                    return self._current_point
+
+                else:
+                    #If none of the neighbours are better then do a random restart
+                    self._current_state = "Not Initialized"
+                    restarting_point = self._generate_random_point(search_space)
+                    self._current_point = restarting_point
+
+                    self._best_neighbor = None
+                    self._best_neighbor_value = None
+
+                    #Add the neighbours
+                    neighbors = self._generate_neighbors(restarting_point, search_space)
+                    self._remaining_points.extend(neighbors)
+
+                    #Change the state to initialized
+                    self._current_state = "Initialized"
+
+                    #Return the current point
+                    return self._current_point
+
+            else:
+                #Process as normal
+                current_point = self._remaining_points.pop()
+                return current_point         
+
+
+if __name__ == "__main__":
+    def objective(trial):
+        x = trial.suggest_float("x", -10, 10, step=1)
+        y = trial.suggest_float("y", -10, 10, step=1)
+        z = trial.suggest_float("z", -10, 10, step=1)
+
+        return x**2+y**2+z
+
+    sampler = HillClimbSearch()
+    study = optuna.create_study(sampler=sampler)
+    study.optimize(objective, n_trials=100)
+
+    print(study.best_trial.value, study.best_trial.params)