[docs][tune] Make search algorithm, scheduler docs better! (#8179)

python/ray/tune/examples/hyperopt_example.py

@@ -28,7 +28,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--smoke-test", action="store_true", help="Finish quickly for testing")
     args, _ = parser.parse_known_args()
-    ray.init()
+    ray.init(configure_logging=False)
 
     space = {
         "width": hp.uniform("width", 0, 20),

python/ray/tune/suggest/ax.py

@@ -10,11 +10,18 @@ logger = logging.getLogger(__name__)
 
 
 class AxSearch(Searcher):
-    """A wrapper around Ax to provide trial suggestions.
+    """Uses `Ax <https://ax.dev/>`_ to optimize hyperparameters.
 
-    Requires Ax to be installed. Ax is an open source tool from
-    Facebook for configuring and optimizing experiments. More information
-    can be found in https://ax.dev/.
+    Ax is a platform for understanding, managing, deploying, and
+    automating adaptive experiments. Ax provides an easy to use
+    interface with BoTorch, a flexible, modern library for Bayesian
+    optimization in PyTorch. More information can be found in https://ax.dev/.
+
+    To use this search algorithm, you must install Ax and sqlalchemy:
+
+    .. code-block:: bash
+
+        $ pip install ax-platform sqlalchemy
 
     Parameters:
         parameters (list[dict]): Parameters in the experiment search space.

python/ray/tune/suggest/bayesopt.py

@@ -12,10 +12,19 @@ logger = logging.getLogger(__name__)
 
 
 class BayesOptSearch(Searcher):
-    """A wrapper around BayesOpt to provide trial suggestions.
+    """Uses fmfn/BayesianOptimization to optimize hyperparameters.
 
-    Requires BayesOpt to be installed. You can install BayesOpt with the
-    command: ``pip install bayesian-optimization``.
+    fmfn/BayesianOptimization is a library for Bayesian Optimization. More
+    info can be found here: https://github.com/fmfn/BayesianOptimization.
+
+    You will need to install fmfn/BayesianOptimization via the following:
+
+    .. code-block:: bash
+
+        pip install bayesian-optimization
+
+    This algorithm requires setting a search space using the
+    `BayesianOptimization search space specification`_.
 
     Parameters:
         space (dict): Continuous search space. Parameters will be sampled from

python/ray/tune/suggest/dragonfly.py

@@ -16,27 +16,26 @@ logger = logging.getLogger(__name__)
 
 
 class DragonflySearch(Searcher):
-    """A wrapper around Dragonfly to provide trial suggestions.
+    """Uses Dragonfly to optimize hyperparameters.
 
-    Requires Dragonfly to be installed via ``pip install dragonfly-opt``.
+    Dragonfly provides an array of tools to scale up Bayesian optimisation to
+    expensive large scale problems, including high dimensional optimisation.
+    parallel evaluations in synchronous or asynchronous settings,
+    multi-fidelity optimisation (using cheap approximations to speed up the
+    optimisation process), and multi-objective optimisation. For more info:
 
-    Parameters:
-        optimizer (dragonfly.opt.BlackboxOptimiser): Optimizer provided
-            from dragonfly. Choose an optimiser that extends BlackboxOptimiser.
-        metric (str): The training result objective value attribute.
-        mode (str): One of {min, max}. Determines whether objective is
-            minimizing or maximizing the metric attribute.
-        points_to_evaluate (list of lists): A list of points you'd like to run
-            first before sampling from the optimiser, e.g. these could be
-            parameter configurations you already know work well to help
-            the optimiser select good values. Each point is a list of the
-            parameters using the order definition given by parameter_names.
-        evaluated_rewards (list): If you have previously evaluated the
-            parameters passed in as points_to_evaluate you can avoid
-            re-running those trials by passing in the reward attributes
-            as a list so the optimiser can be told the results without
-            needing to re-compute the trial. Must be the same length as
-            points_to_evaluate.
+    * Dragonfly Website: https://github.com/dragonfly/dragonfly
+    * Dragonfly Documentation: https://dragonfly-opt.readthedocs.io/
+
+    To use this search algorithm, install Dragonfly:
+
+    .. code-block:: bash
+
+        $ pip install dragonfly-opt
+
+
+    This interface requires using FunctionCallers and optimizers provided by
+    Dragonfly.
 
     .. code-block:: python
 
@@ -70,6 +69,25 @@ class DragonflySearch(Searcher):
         algo = DragonflySearch(optimizer, metric="objective", mode="max")
 
         tune.run(my_func, algo=algo)
+
+    Parameters:
+        optimizer (dragonfly.opt.BlackboxOptimiser): Optimizer provided
+            from dragonfly. Choose an optimiser that extends BlackboxOptimiser.
+        metric (str): The training result objective value attribute.
+        mode (str): One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        points_to_evaluate (list of lists): A list of points you'd like to run
+            first before sampling from the optimiser, e.g. these could be
+            parameter configurations you already know work well to help
+            the optimiser select good values. Each point is a list of the
+            parameters using the order definition given by parameter_names.
+        evaluated_rewards (list): If you have previously evaluated the
+            parameters passed in as points_to_evaluate you can avoid
+            re-running those trials by passing in the reward attributes
+            as a list so the optimiser can be told the results without
+            needing to re-compute the trial. Must be the same length as
+            points_to_evaluate.
+
     """
 
     def __init__(self,

python/ray/tune/suggest/hyperopt.py

@@ -19,12 +19,42 @@ logger = logging.getLogger(__name__)
 class HyperOptSearch(Searcher):
     """A wrapper around HyperOpt to provide trial suggestions.
 
-    Requires HyperOpt to be installed from source.
-    Uses the Tree-structured Parzen Estimators algorithm, although can be
-    trivially extended to support any algorithm HyperOpt uses. Externally
-    added trials will not be tracked by HyperOpt. Trials of the current run
-    can be saved using save method, trials of a previous run can be loaded
-    using restore method, thus enabling a warm start feature.
+    HyperOpt a Python library for serial and parallel optimization
+    over awkward search spaces, which may include real-valued, discrete,
+    and conditional dimensions. More info can be found at
+    http://hyperopt.github.io/hyperopt.
+
+    HyperOptSearch uses the Tree-structured Parzen Estimators algorithm,
+    though it can be trivially extended to support any algorithm HyperOpt
+    supports.
+
+    To use this search algorithm, you will need to install HyperOpt:
+
+    .. code-block:: bash
+
+        pip install -U hyperopt
+
+    You will not be able to leverage Tune's default ``grid_search``
+    and random search primitives when using HyperOptSearch. You need to
+    use the `HyperOpt search space specification
+    <https://github.com/hyperopt/hyperopt/wiki/FMin>`_.
+
+    .. code-block:: python
+
+        space = {
+            'width': hp.uniform('width', 0, 20),
+            'height': hp.uniform('height', -100, 100),
+            'activation': hp.choice("activation", ["relu", "tanh"])
+        }
+        current_best_params = [{
+            'width': 10,
+            'height': 0,
+            'activation': 0, # The index of "relu"
+        }]
+        algo = HyperOptSearch(
+            space, metric="mean_loss", mode="min",
+            points_to_evaluate=current_best_params)
+
 
     Parameters:
         space (dict): HyperOpt configuration. Parameters will be sampled
@@ -50,22 +80,6 @@ class HyperOptSearch(Searcher):
         max_concurrent: Deprecated.
         use_early_stopped_trials: Deprecated.
 
-    .. code-block:: python
-
-        space = {
-            'width': hp.uniform('width', 0, 20),
-            'height': hp.uniform('height', -100, 100),
-            'activation': hp.choice("activation", ["relu", "tanh"])
-        }
-        current_best_params = [{
-            'width': 10,
-            'height': 0,
-            'activation': 0, # The index of "relu"
-        }]
-        algo = HyperOptSearch(
-            space, metric="mean_loss", mode="min",
-            points_to_evaluate=current_best_params)
-
     """
 
     def __init__(

python/ray/tune/suggest/nevergrad.py

@@ -11,14 +11,30 @@ logger = logging.getLogger(__name__)
 
 
 class NevergradSearch(Searcher):
-    """A wrapper around Nevergrad to provide trial suggestions.
-
-    Requires Nevergrad to be installed.
+    """Uses Nevergrad to optimize hyperparameters.
 
     Nevergrad is an open source tool from Facebook for derivative free
-    optimization of parameters and/or hyperparameters. It features a wide
-    range of optimizers in a standard ask and tell interface. More information
-    can be found at https://github.com/facebookresearch/nevergrad.
+    optimization.  More info can be found at:
+    https://github.com/facebookresearch/nevergrad.
+
+    You will need to install Nevergrad via the following command:
+
+    .. code-block:: bash
+
+        $ pip install nevergrad
+
+    This algorithm requires using an optimizer provided by Nevergrad, of
+    which there are many options. A good rundown can be found on
+    the `Nevergrad README's Optimization section`_.
+
+    .. code-block:: python
+
+        from nevergrad.optimization import optimizerlib
+
+        instrumentation = 1
+        optimizer = optimizerlib.OnePlusOne(instrumentation, budget=100)
+        algo = NevergradSearch(
+            optimizer, ["lr"], metric="mean_loss", mode="min")
 
     Parameters:
         optimizer (nevergrad.optimization.Optimizer): Optimizer provided
@@ -33,15 +49,6 @@ class NevergradSearch(Searcher):
         use_early_stopped_trials: Deprecated.
         max_concurrent: Deprecated.
 
-    .. code-block:: python
-
-        from nevergrad.optimization import optimizerlib
-
-        instrumentation = 1
-        optimizer = optimizerlib.OnePlusOne(instrumentation, budget=100)
-        algo = NevergradSearch(
-            optimizer, ["lr"], metric="mean_loss", mode="min")
-
     Note:
         In nevergrad v0.2.0+, optimizers can be instrumented.
         For instance, the following will specifies searching

python/ray/tune/suggest/repeater.py

@@ -88,6 +88,18 @@ class Repeater(Searcher):
             Trainable/Function config which corresponds to the index of the
             repeated trial. This can be used for seeds. Defaults to True.
 
+    Example:
+
+    .. code-block:: python
+
+        from ray.tune.suggest import Repeater
+
+        search_alg = BayesOptSearch(...)
+        re_search_alg = Repeater(search_alg, repeat=10)
+
+        # Repeat 2 samples 10 times each.
+        tune.run(trainable, num_samples=20, search_alg=re_search_alg)
+
     """
 
     def __init__(self, searcher, repeat=1, set_index=True):

python/ray/tune/suggest/sigopt.py

@@ -15,8 +15,16 @@ logger = logging.getLogger(__name__)
 class SigOptSearch(Searcher):
     """A wrapper around SigOpt to provide trial suggestions.
 
-    Requires SigOpt to be installed. Requires user to store their SigOpt
-    API key locally as an environment variable at `SIGOPT_KEY`.
+    You must install SigOpt and have a SigOpt API key to use this module.
+    Store the API token as an environment variable ``SIGOPT_KEY`` as follows:
+
+    .. code-block:: bash
+
+        pip install -U sigopt
+        export SIGOPT_KEY= ...
+
+    You will need to use the `SigOpt experiment and space specification
+    <https://app.sigopt.com/docs/overview/create>`_.
 
     This module manages its own concurrency.
 

python/ray/tune/suggest/skopt.py

@@ -41,9 +41,19 @@ def _validate_warmstart(parameter_names, points_to_evaluate,
 
 
 class SkOptSearch(Searcher):
-    """A wrapper around skopt to provide trial suggestions.
+    """Uses Scikit Optimize (skopt) to optimize hyperparameters.
 
-    Requires skopt to be installed.
+    Scikit-optimize is a black-box optimization library.
+    Read more here: https://scikit-optimize.github.io.
+
+    You will need to install Scikit-Optimize to use this module.
+
+    .. code-block:: bash
+
+        pip install scikit-optimize
+
+
+    This Search Algorithm requires you to pass in a `skopt Optimizer object`_.
 
     Parameters:
         optimizer (skopt.optimizer.Optimizer): Optimizer provided
@@ -68,14 +78,18 @@ class SkOptSearch(Searcher):
         use_early_stopped_trials: Deprecated.
 
     Example:
-        >>> from skopt import Optimizer
-        >>> optimizer = Optimizer([(0,20),(-100,100)])
-        >>> current_best_params = [[10, 0], [15, -20]]
-        >>> algo = SkOptSearch(optimizer,
-        >>>     ["width", "height"],
-        >>>     metric="mean_loss",
-        >>>     mode="min",
-        >>>     points_to_evaluate=current_best_params)
+
+    .. code-block:: python
+
+        from skopt import Optimizer
+        optimizer = Optimizer([(0,20),(-100,100)])
+        current_best_params = [[10, 0], [15, -20]]
+
+        algo = SkOptSearch(optimizer,
+            ["width", "height"],
+            metric="mean_loss",
+            mode="min",
+            points_to_evaluate=current_best_params)
     """
 
     def __init__(self,

python/ray/tune/suggest/zoopt.py

@@ -15,22 +15,12 @@ logger = logging.getLogger(__name__)
 class ZOOptSearch(Searcher):
     """A wrapper around ZOOpt to provide trial suggestions.
 
-    Requires zoopt package (>=0.4.0) to be installed. You can install it
-    with the command: ``pip install -U zoopt``.
+    ZOOptSearch is a library for derivative-free optimization. It is backed by
+    the `ZOOpt <https://github.com/polixir/ZOOpt>`__ package. Currently,
+    Asynchronous Sequential RAndomized COordinate Shrinking (ASRacos)
+    is implemented in Tune.
 
-    Parameters:
-        algo (str): To specify an algorithm in zoopt you want to use.
-            Only support ASRacos currently.
-        budget (int): Number of samples.
-        dim_dict (dict): Dimension dictionary.
-            For continuous dimensions: (continuous, search_range, precision);
-            For discrete dimensions: (discrete, search_range, has_order).
-            More details can be found in zoopt package.
-        metric (str): The training result objective value attribute.
-            Defaults to "episode_reward_mean".
-        mode (str): One of {min, max}. Determines whether objective is
-            minimizing or maximizing the metric attribute.
-            Defaults to "min".
+    To use ZOOptSearch, install zoopt (>=0.4.0): ``pip install -U zoopt``.
 
     .. code-block:: python
 
@@ -65,6 +55,20 @@ class ZOOptSearch(Searcher):
             name="zoopt_search",
             **config)
 
+    Parameters:
+        algo (str): To specify an algorithm in zoopt you want to use.
+            Only support ASRacos currently.
+        budget (int): Number of samples.
+        dim_dict (dict): Dimension dictionary.
+            For continuous dimensions: (continuous, search_range, precision);
+            For discrete dimensions: (discrete, search_range, has_order).
+            More details can be found in zoopt package.
+        metric (str): The training result objective value attribute.
+            Defaults to "episode_reward_mean".
+        mode (str): One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+            Defaults to "min".
+
     """
 
     optimizer = None