From e311013afd4b823724700c5ca6847b51e48f6697 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Mon, 23 Mar 2020 12:23:21 -0700 Subject: [PATCH] [tune] Reformat Sections of API Reference (#7706) * moveit * moveit * docstrings to ref * Update tune-usage.rst Co-authored-by: Sven Mika --- doc/source/index.rst | 3 +- doc/source/tune-package-ref.rst | 302 ------------------ doc/source/tune-searchalg.rst | 4 +- doc/source/tune-usage.rst | 161 +--------- doc/source/tune/api_docs/analysis.rst | 58 ++++ doc/source/tune/api_docs/cli.rst | 39 +++ doc/source/tune/api_docs/client.rst | 36 +++ doc/source/tune/api_docs/execution.rst | 24 ++ doc/source/tune/api_docs/grid_random.rst | 46 +++ .../api_docs/internals.rst} | 65 +++- doc/source/tune/api_docs/overview.rst | 21 ++ doc/source/tune/api_docs/reporters.rst | 92 ++++++ doc/source/tune/api_docs/schedulers.rst | 36 +++ doc/source/tune/api_docs/suggestion.rst | 63 ++++ doc/source/tune/api_docs/trainable.rst | 32 ++ 15 files changed, 514 insertions(+), 468 deletions(-) delete mode 100644 doc/source/tune-package-ref.rst create mode 100644 doc/source/tune/api_docs/analysis.rst create mode 100644 doc/source/tune/api_docs/cli.rst create mode 100644 doc/source/tune/api_docs/client.rst create mode 100644 doc/source/tune/api_docs/execution.rst create mode 100644 doc/source/tune/api_docs/grid_random.rst rename doc/source/{tune-design.rst => tune/api_docs/internals.rst} (76%) create mode 100644 doc/source/tune/api_docs/overview.rst create mode 100644 doc/source/tune/api_docs/reporters.rst create mode 100644 doc/source/tune/api_docs/schedulers.rst create mode 100644 doc/source/tune/api_docs/suggestion.rst create mode 100644 doc/source/tune/api_docs/trainable.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index fcea62ff0..cab0ec0f7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -254,9 +254,8 @@ Getting Involved tune-distributed.rst tune-schedulers.rst tune-searchalg.rst - tune-design.rst tune-examples.rst - tune-package-ref.rst + tune/api_docs/overview.rst tune-contrib.rst .. toctree:: diff --git a/doc/source/tune-package-ref.rst b/doc/source/tune-package-ref.rst deleted file mode 100644 index 8c867b0a0..000000000 --- a/doc/source/tune-package-ref.rst +++ /dev/null @@ -1,302 +0,0 @@ -Tune Package Reference -======================= - -Training (tune.run, tune.Experiment) ------------------------------------- - -tune.run -~~~~~~~~ - -.. autofunction:: ray.tune.run - -tune.run_experiments -~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: ray.tune.run_experiments - -tune.Experiment -~~~~~~~~~~~~~~~ - -.. autofunction:: ray.tune.Experiment - -Trainable (tune.Trainable, tune.track) --------------------------------------- - -tune.Trainable -~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.Trainable - :member-order: groupwise - :private-members: - :members: - -tune.DurableTrainable -~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.DurableTrainable - -tune.track -~~~~~~~~~~ - -.. automodule:: ray.tune.track - :members: - :exclude-members: init, shutdown - -StatusReporter -~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.function_runner.StatusReporter - :members: __call__, logdir - -Sampling (tune.rand, tune.grid_search...) ------------------------------------------ - -tune.randn -~~~~~~~~~~ - -.. autofunction:: ray.tune.randn - -tune.loguniform -~~~~~~~~~~~~~~~ - -.. autofunction:: ray.tune.loguniform - -tune.uniform -~~~~~~~~~~~~ - -.. autofunction:: ray.tune.uniform - -tune.choice -~~~~~~~~~~~ - -.. autofunction:: ray.tune.choice - -tune.sample_from -~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.sample_from - -tune.grid_search -~~~~~~~~~~~~~~~~ - -.. autofunction:: ray.tune.grid_search - -Stopper (tune.Stopper) ----------------------- - -.. autoclass:: ray.tune.Stopper - :members: __call__, stop_all - -Analysis (tune.analysis) ------------------------- - -ExperimentAnalysis -~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.ExperimentAnalysis - :show-inheritance: - :members: - - -Analysis -~~~~~~~~ - -.. autoclass:: ray.tune.Analysis - :members: - - -Schedulers (tune.schedulers) ----------------------------- - -FIFOScheduler -~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.schedulers.FIFOScheduler - -HyperBandScheduler -~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.schedulers.HyperBandScheduler - -ASHAScheduler/AsyncHyperBandScheduler -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.schedulers.AsyncHyperBandScheduler - -.. autoclass:: ray.tune.schedulers.ASHAScheduler - -MedianStoppingRule -~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.schedulers.MedianStoppingRule - -PopulationBasedTraining -~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.schedulers.PopulationBasedTraining - - -TrialScheduler -~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.schedulers.TrialScheduler - :members: - - -Search Algorithms (tune.suggest) --------------------------------- - -BasicVariantGenerator -~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.suggest.BasicVariantGenerator - -AxSearch -~~~~~~~~ - -.. autoclass:: ray.tune.suggest.ax.AxSearch - -BayesOptSearch -~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.suggest.bayesopt.BayesOptSearch - -TuneBOHB -~~~~~~~~ - -.. autoclass:: ray.tune.suggest.bohb.TuneBOHB - -DragonflySearch -~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.suggest.dragonfly.DragonflySearch - -HyperOptSearch -~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.suggest.hyperopt.HyperOptSearch - -NevergradSearch -~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.suggest.nevergrad.NevergradSearch - -SigOptSearch -~~~~~~~~~~~~ - -.. autoclass:: ray.tune.suggest.sigopt.SigOptSearch - -SkOptSearch -~~~~~~~~~~~ - -.. autoclass:: ray.tune.suggest.skopt.SkOptSearch - -SearchAlgorithm -~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.suggest.SearchAlgorithm - :members: - -SuggestionAlgorithm -~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.suggest.SuggestionAlgorithm - :members: - :private-members: - :show-inheritance: - -Repeater -~~~~~~~~ - -.. autoclass:: ray.tune.suggest.Repeater - -Loggers (tune.logger) ---------------------- - -Logger -~~~~~~ - -.. autoclass:: ray.tune.logger.Logger - -UnifiedLogger -~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.logger.UnifiedLogger - -TBXLogger -~~~~~~~~~ - -.. autoclass:: ray.tune.logger.TBXLogger - -JsonLogger -~~~~~~~~~~ - -.. autoclass:: ray.tune.logger.JsonLogger - -CSVLogger -~~~~~~~~~ - -.. autoclass:: ray.tune.logger.CSVLogger - -MLFLowLogger -~~~~~~~~~~~~ - -.. autoclass:: ray.tune.logger.MLFLowLogger - - -Reporters ---------- - -ProgressReporter -~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.ProgressReporter - :members: - -CLIReporter -~~~~~~~~~~~ - -.. autoclass:: ray.tune.CLIReporter - -JupyterNotebookReporter -~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.JupyterNotebookReporter - - -Internals ---------- - -Registry -~~~~~~~~ - -.. autofunction:: ray.tune.register_trainable - -.. autofunction:: ray.tune.register_env - -RayTrialExecutor -~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.ray_trial_executor.RayTrialExecutor - :members: - -TrialExecutor -~~~~~~~~~~~~~ - -.. autoclass:: ray.tune.trial_executor.TrialExecutor - :members: - -TrialRunner -~~~~~~~~~~~ - -.. autoclass:: ray.tune.trial_runner.TrialRunner - -Trial -~~~~~ - -.. autoclass:: ray.tune.trial.Trial - -Resources -~~~~~~~~~ - -.. autoclass:: ray.tune.resources.Resources diff --git a/doc/source/tune-searchalg.rst b/doc/source/tune-searchalg.rst index 6318e2c99..3dae5b8b0 100644 --- a/doc/source/tune-searchalg.rst +++ b/doc/source/tune-searchalg.rst @@ -46,7 +46,7 @@ The ``search_alg`` will suggest new configurations to try, and the ``Repeater`` will run ``repeat`` trials of the configuration. It will then average the ``search_alg.metric`` from the final results of each repeated trial. -See `Repeater `_ docstring for more details. +See the API documentation (:ref:`repeater-doc`) for more details. .. code-block:: python @@ -172,7 +172,7 @@ In order to use this search algorithm, you will need to install Scikit-Optimize $ pip install scikit-optimize -This algorithm requires using the `Scikit-Optimize ask and tell interface `__. This interface requires using the `Optimizer `__ provided by Scikit-Optimize. You can use SkOptSearch like follows: +This algorithm requires using the `Scikit-Optimize ask and tell interface `__. This interface requires using the `Optimizer `__ provided by Scikit-Optimize. You can use SkOptSearch like follows: .. code-block:: python diff --git a/doc/source/tune-usage.rst b/doc/source/tune-usage.rst index da8e61ea7..fe16a251f 100644 --- a/doc/source/tune-usage.rst +++ b/doc/source/tune-usage.rst @@ -25,7 +25,7 @@ Training can be done with either the Trainable **Class API** or **function-based Trainable API ~~~~~~~~~~~~~ -The class-based API will require users to subclass ``ray.tune.Trainable``. The Trainable interface `can be found here `__. +The class-based API will require users to subclass ``ray.tune.Trainable``. See the API documentation: :ref:`trainable-docstring`. Here is an example: @@ -63,7 +63,7 @@ User-defined functions will need to have following signature and call ``tune.tra tune.track.log(**kwargs) -Tune will run this function on a separate thread in a Ray actor process. Note that this API is not checkpointable, since the thread will never return control back to its caller. ``tune.track`` documentation can be `found here `__. +Tune will run this function on a separate thread in a Ray actor process. Note that this API is not checkpointable, since the thread will never return control back to its caller. ``tune.track`` documentation can be found here: :ref:`track-docstring`. Both the Trainable and function-based API will have `autofilled metrics `__ in addition to the metrics reported. @@ -185,7 +185,7 @@ You may want to get a summary of multiple experiments that point to the same ``l from ray.tune import Analysis analysis = Analysis("~/ray_results/example-experiment") -See the `full documentation `_ for the ``Analysis`` object. +See the full documentation for the ``Analysis`` object: :ref:`analysis-docstring`. Tune Search Space (Default) @@ -209,7 +209,7 @@ Use ``tune.sample_from()`` to sample a value for a hyperparameter. The ``f } ) -Tune provides a couple helper functions for common parameter distributions, wrapping numpy random utilities such as ``np.random.uniform``, ``np.random.choice``, and ``np.random.randn``. See the `Package Reference `_ for more details. +Tune provides a couple of helper functions for common parameter distributions, wrapping numpy random utilities such as ``np.random.uniform``, ``np.random.choice``, and ``np.random.randn``. See :ref:`tune-sample-docs` for more details. The following shows grid search over two nested parameters combined with random sampling from two lambda functions, generating 9 different trials. Note that the value of ``beta`` depends on the value of ``alpha``, which is represented by referencing ``spec.config.alpha`` in the lambda function. This lets you specify conditional parameter distributions. @@ -616,7 +616,7 @@ You can pass in your own logging mechanisms to output logs in custom formats as loggers=DEFAULT_LOGGERS + (CustomLogger1, CustomLogger2) ) -These loggers will be called along with the default Tune loggers. All loggers must inherit the `Logger interface `__. Tune enables default loggers for Tensorboard, CSV, and JSON formats. You can also check out `logger.py `__ for implementation details. An example can be found in `logging_example.py `__. +These loggers will be called along with the default Tune loggers. All loggers must inherit the Logger interface (:ref:`logger-interface`). Tune enables default loggers for Tensorboard, CSV, and JSON formats. You can also check out `logger.py `__ for implementation details. An example can be found in `logging_example.py `__. MLFlow ~~~~~~ @@ -660,42 +660,6 @@ You can customize this to specify arbitrary storages with the ``sync_to_cloud`` sync_to_cloud=custom_sync_func, ) -Tune Client API ---------------- - -You can interact with an ongoing experiment with the Tune Client API. The Tune Client API is organized around REST, which includes resource-oriented URLs, accepts form-encoded requests, returns JSON-encoded responses, and uses standard HTTP protocol. - -To allow Tune to receive and respond to your API calls, you have to start your experiment with ``with_server=True``: - -.. code-block:: python - - tune.run(..., with_server=True, server_port=4321) - -The easiest way to use the Tune Client API is with the built-in TuneClient. To use TuneClient, verify that you have the ``requests`` library installed: - -.. code-block:: bash - - $ pip install requests - -Then, on the client side, you can use the following class. If on a cluster, you may want to forward this port (e.g. ``ssh -L :localhost:
``) so that you can use the Client on your local machine. - -.. autoclass:: ray.tune.web_server.TuneClient - :members: - -For an example notebook for using the Client API, see the `Client API Example `__. - -The API also supports curl. Here are the examples for getting trials (``GET /trials/[:id]``): - -.. code-block:: bash - - $ curl http://
:/trials - $ curl http://
:/trials/ - -And stopping a trial (``PUT /trials/:id``): - -.. code-block:: bash - - $ curl -X PUT http://
:/trials/ Debugging --------- @@ -708,121 +672,6 @@ By default, Tune will run hyperparameter evaluations on multiple processes. Howe Note that some behavior such as writing to files by depending on the current working directory in a Trainable and setting global process variables may not work as expected. Local mode with multiple configuration evaluations will interleave computation, so it is most naturally used when running a single configuration evaluation. -CLI Progress Reporting ----------------------- - -By default, Tune reports experiment progress periodically to the command-line as follows. - -.. code-block:: bash - - == Status == - Memory usage on this node: 11.4/16.0 GiB - Using FIFO scheduling algorithm. - Resources requested: 4/12 CPUs, 0/0 GPUs, 0.0/3.17 GiB heap, 0.0/1.07 GiB objects - Result logdir: /Users/foo/ray_results/myexp - Number of trials: 4 (4 RUNNING) - +----------------------+----------+---------------------+-----------+--------+--------+--------+--------+------------------+-------+ - | Trial name | status | loc | param1 | param2 | param3 | acc | loss | total time (s) | iter | - |----------------------+----------+---------------------+-----------+--------+--------+--------+--------+------------------+-------| - | MyTrainable_a826033a | RUNNING | 10.234.98.164:31115 | 0.303706 | 0.0761 | 0.4328 | 0.1289 | 1.8572 | 7.54952 | 15 | - | MyTrainable_a8263fc6 | RUNNING | 10.234.98.164:31117 | 0.929276 | 0.158 | 0.3417 | 0.4865 | 1.6307 | 7.0501 | 14 | - | MyTrainable_a8267914 | RUNNING | 10.234.98.164:31111 | 0.068426 | 0.0319 | 0.1147 | 0.9585 | 1.9603 | 7.0477 | 14 | - | MyTrainable_a826b7bc | RUNNING | 10.234.98.164:31112 | 0.729127 | 0.0748 | 0.1784 | 0.1797 | 1.7161 | 7.05715 | 14 | - +----------------------+----------+---------------------+-----------+--------+--------+--------+--------+------------------+-------+ - -Note that columns will be hidden if they are completely empty. The output can be configured in various ways by instantiating a ``CLIReporter`` instance (or ``JupyterNotebookReporter`` if you're using jupyter notebook). Here's an example: - -.. code-block:: python - - from ray.tune import CLIReporter - - # Limit the number of rows. - reporter = CLIReporter(max_progress_rows=10) - # Add a custom metric column, in addition to the default metrics. - # Note that this must be a metric that is returned in your training results. - reporter.add_metric_column("custom_metric") - tune.run(my_trainable, progress_reporter=reporter) - -Extending ``CLIReporter`` lets you control reporting frequency. For example: - -.. code-block:: python - - class ExperimentTerminationReporter(CLIReporter): - def should_report(self, trials, done=False): - """Reports only on experiment termination.""" - return done - - tune.run(my_trainable, progress_reporter=ExperimentTerminationReporter()) - - class TrialTerminationReporter(CLIReporter): - def __init__(self): - self.num_terminated = 0 - - def should_report(self, trials, done=False): - """Reports only on trial termination events.""" - old_num_terminated = self.num_terminated - self.num_terminated = len([t for t in trials if t.status == Trial.TERMINATED]) - return self.num_terminated > old_num_terminated - - tune.run(my_trainable, progress_reporter=TrialTerminationReporter()) - -The default reporting style can also be overriden more broadly by extending the ``ProgressReporter`` interface directly. Note that you can print to any output stream, file etc. - -.. code-block:: python - - from ray.tune import ProgressReporter - - class CustomReporter(ProgressReporter): - - def should_report(self, trials, done=False): - return True - - def report(self, trials, *sys_info): - print(*sys_info) - print("\n".join([str(trial) for trial in trials])) - - tune.run(my_trainable, progress_reporter=CustomReporter()) - -Tune CLI (Experimental) ------------------------ - -``tune`` has an easy-to-use command line interface (CLI) to manage and monitor your experiments on Ray. To do this, verify that you have the ``tabulate`` library installed: - -.. code-block:: bash - - $ pip install tabulate - -Here are a few examples of command line calls. - -- ``tune list-trials``: List tabular information about trials within an experiment. Empty columns will be dropped by default. Add the ``--sort`` flag to sort the output by specific columns. Add the ``--filter`` flag to filter the output in the format ``" "``. Add the ``--output`` flag to write the trial information to a specific file (CSV or Pickle). Add the ``--columns`` and ``--result-columns`` flags to select specific columns to display. - -.. code-block:: bash - - $ tune list-trials [EXPERIMENT_DIR] --output note.csv - - +------------------+-----------------------+------------+ - | trainable_name | experiment_tag | trial_id | - |------------------+-----------------------+------------| - | MyTrainableClass | 0_height=40,width=37 | 87b54a1d | - | MyTrainableClass | 1_height=21,width=70 | 23b89036 | - | MyTrainableClass | 2_height=99,width=90 | 518dbe95 | - | MyTrainableClass | 3_height=54,width=21 | 7b99a28a | - | MyTrainableClass | 4_height=90,width=69 | ae4e02fb | - +------------------+-----------------------+------------+ - Dropped columns: ['status', 'last_update_time'] - Please increase your terminal size to view remaining columns. - Output saved at: note.csv - - $ tune list-trials [EXPERIMENT_DIR] --filter "trial_id == 7b99a28a" - - +------------------+-----------------------+------------+ - | trainable_name | experiment_tag | trial_id | - |------------------+-----------------------+------------| - | MyTrainableClass | 3_height=54,width=21 | 7b99a28a | - +------------------+-----------------------+------------+ - Dropped columns: ['status', 'last_update_time'] - Please increase your terminal size to view remaining columns. - Further Questions or Issues? ---------------------------- diff --git a/doc/source/tune/api_docs/analysis.rst b/doc/source/tune/api_docs/analysis.rst new file mode 100644 index 000000000..dd516ad82 --- /dev/null +++ b/doc/source/tune/api_docs/analysis.rst @@ -0,0 +1,58 @@ +Analysis/Logging (tune.analysis / tune.logger) +============================================== + +Analyzing Results +----------------- + +ExperimentAnalysis +~~~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.ExperimentAnalysis + :show-inheritance: + :members: + +.. _analysis-docstring: + +Analysis +~~~~~~~~ + +.. autoclass:: ray.tune.Analysis + :members: + +.. _loggers-docstring: + +Loggers (tune.logger) +--------------------- + +.. _logger-interface: + +Logger +~~~~~~ + +.. autoclass:: ray.tune.logger.Logger + +UnifiedLogger +~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.logger.UnifiedLogger + +TBXLogger +~~~~~~~~~ + +.. autoclass:: ray.tune.logger.TBXLogger + +JsonLogger +~~~~~~~~~~ + +.. autoclass:: ray.tune.logger.JsonLogger + +CSVLogger +~~~~~~~~~ + +.. autoclass:: ray.tune.logger.CSVLogger + +MLFLowLogger +~~~~~~~~~~~~ + +.. autoclass:: ray.tune.logger.MLFLowLogger + diff --git a/doc/source/tune/api_docs/cli.rst b/doc/source/tune/api_docs/cli.rst new file mode 100644 index 000000000..629ea97f1 --- /dev/null +++ b/doc/source/tune/api_docs/cli.rst @@ -0,0 +1,39 @@ +Tune CLI (Experimental) +======================= + +``tune`` has an easy-to-use command line interface (CLI) to manage and monitor your experiments on Ray. To do this, verify that you have the ``tabulate`` library installed: + +.. code-block:: bash + + $ pip install tabulate + +Here are a few examples of command line calls. + +- ``tune list-trials``: List tabular information about trials within an experiment. Empty columns will be dropped by default. Add the ``--sort`` flag to sort the output by specific columns. Add the ``--filter`` flag to filter the output in the format ``" "``. Add the ``--output`` flag to write the trial information to a specific file (CSV or Pickle). Add the ``--columns`` and ``--result-columns`` flags to select specific columns to display. + +.. code-block:: bash + + $ tune list-trials [EXPERIMENT_DIR] --output note.csv + + +------------------+-----------------------+------------+ + | trainable_name | experiment_tag | trial_id | + |------------------+-----------------------+------------| + | MyTrainableClass | 0_height=40,width=37 | 87b54a1d | + | MyTrainableClass | 1_height=21,width=70 | 23b89036 | + | MyTrainableClass | 2_height=99,width=90 | 518dbe95 | + | MyTrainableClass | 3_height=54,width=21 | 7b99a28a | + | MyTrainableClass | 4_height=90,width=69 | ae4e02fb | + +------------------+-----------------------+------------+ + Dropped columns: ['status', 'last_update_time'] + Please increase your terminal size to view remaining columns. + Output saved at: note.csv + + $ tune list-trials [EXPERIMENT_DIR] --filter "trial_id == 7b99a28a" + + +------------------+-----------------------+------------+ + | trainable_name | experiment_tag | trial_id | + |------------------+-----------------------+------------| + | MyTrainableClass | 3_height=54,width=21 | 7b99a28a | + +------------------+-----------------------+------------+ + Dropped columns: ['status', 'last_update_time'] + Please increase your terminal size to view remaining columns. diff --git a/doc/source/tune/api_docs/client.rst b/doc/source/tune/api_docs/client.rst new file mode 100644 index 000000000..775f3c077 --- /dev/null +++ b/doc/source/tune/api_docs/client.rst @@ -0,0 +1,36 @@ +Tune Client API +=============== + +You can interact with an ongoing experiment with the Tune Client API. The Tune Client API is organized around REST, which includes resource-oriented URLs, accepts form-encoded requests, returns JSON-encoded responses, and uses standard HTTP protocol. + +To allow Tune to receive and respond to your API calls, you have to start your experiment with ``with_server=True``: + +.. code-block:: python + + tune.run(..., with_server=True, server_port=4321) + +The easiest way to use the Tune Client API is with the built-in TuneClient. To use TuneClient, verify that you have the ``requests`` library installed: + +.. code-block:: bash + + $ pip install requests + +Then, on the client side, you can use the following class. If on a cluster, you may want to forward this port (e.g. ``ssh -L :localhost:
``) so that you can use the Client on your local machine. + +.. autoclass:: ray.tune.web_server.TuneClient + :members: + +For an example notebook for using the Client API, see the `Client API Example `__. + +The API also supports curl. Here are the examples for getting trials (``GET /trials/[:id]``): + +.. code-block:: bash + + $ curl http://
:/trials + $ curl http://
:/trials/ + +And stopping a trial (``PUT /trials/:id``): + +.. code-block:: bash + + $ curl -X PUT http://
:/trials/ diff --git a/doc/source/tune/api_docs/execution.rst b/doc/source/tune/api_docs/execution.rst new file mode 100644 index 000000000..3b82d3256 --- /dev/null +++ b/doc/source/tune/api_docs/execution.rst @@ -0,0 +1,24 @@ +Training (tune.run, tune.Experiment) +==================================== + +tune.run +-------- + +.. autofunction:: ray.tune.run + +tune.run_experiments +-------------------- + +.. autofunction:: ray.tune.run_experiments + +tune.Experiment +--------------- + +.. autofunction:: ray.tune.Experiment + + +Stopper (tune.Stopper) +---------------------- + +.. autoclass:: ray.tune.Stopper + :members: __call__, stop_all diff --git a/doc/source/tune/api_docs/grid_random.rst b/doc/source/tune/api_docs/grid_random.rst new file mode 100644 index 000000000..4d01b0a66 --- /dev/null +++ b/doc/source/tune/api_docs/grid_random.rst @@ -0,0 +1,46 @@ +Grid/Random Search +================== + +.. _tune-sample-docs: + +Random Distributions +-------------------- + +tune.randn +~~~~~~~~~~ + +.. autofunction:: ray.tune.randn + +tune.loguniform +~~~~~~~~~~~~~~~ + +.. autofunction:: ray.tune.loguniform + +tune.uniform +~~~~~~~~~~~~ + +.. autofunction:: ray.tune.uniform + +tune.choice +~~~~~~~~~~~ + +.. autofunction:: ray.tune.choice + +tune.sample_from +~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.sample_from + +Grid Search +----------- + +tune.grid_search +~~~~~~~~~~~~~~~~ + +.. autofunction:: ray.tune.grid_search + + +BasicVariantGenerator +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.suggest.BasicVariantGenerator diff --git a/doc/source/tune-design.rst b/doc/source/tune/api_docs/internals.rst similarity index 76% rename from doc/source/tune-design.rst rename to doc/source/tune/api_docs/internals.rst index b8bf09a08..644a0f42b 100644 --- a/doc/source/tune-design.rst +++ b/doc/source/tune/api_docs/internals.rst @@ -1,13 +1,11 @@ -Tune Design Guide -================= +Tune Internals +============== -In this part of the documentation, we overview the design and architecture -of Tune. +This page overviews the design and architectures of Tune and provides docstrings for internal components. -.. image:: images/tune-arch.png +.. image:: ../../images/tune-arch.png The blue boxes refer to internal components, and green boxes are public-facing. -Please refer to the package reference for `user-facing APIs `__. Main Components --------------- @@ -32,6 +30,8 @@ The TrialRunner is also in charge of checkpointing the entire experiment executi upon each loop iteration. This allows users to restart their experiment in case of machine failure. +See the docstring at :ref:`trialrunner-docstring`. + Trial objects ~~~~~~~~~~~~~ [`source code `__] @@ -41,12 +41,17 @@ distributed/remote. Trial objects transition among the following states: ``"PENDING"``, ``"RUNNING"``, ``"PAUSED"``, ``"ERRORED"``, and ``"TERMINATED"``. +See the docstring at :ref:`trial-docstring`. + TrialExecutor ~~~~~~~~~~~~~ [`source code `__] The TrialExecutor is a component that interacts with the underlying execution framework. It also manages resources to ensure the cluster isn't overloaded. By default, the TrialExecutor uses Ray to execute trials. +See the docstring at :ref:`raytrialexecutor-docstring`. + + SearchAlg ~~~~~~~~~ [`source code `__] The SearchAlgorithm is a user-provided object @@ -73,3 +78,51 @@ Trainable interface. If a function is provided. it is wrapped into a Trainable class, and the function itself is executed on a separate thread. Trainables will execute one step of ``train()`` before notifying the TrialRunner. + + +.. _raytrialexecutor-docstring: + +RayTrialExecutor +---------------- + +.. autoclass:: ray.tune.ray_trial_executor.RayTrialExecutor + :show-inheritance: + :members: + +.. _trialexecutor-docstring: + +TrialExecutor +------------- + +.. autoclass:: ray.tune.trial_executor.TrialExecutor + :members: + +.. _trialrunner-docstring: + +TrialRunner +----------- + +.. autoclass:: ray.tune.trial_runner.TrialRunner + +.. _trial-docstring: + +Trial +----- + +.. autoclass:: ray.tune.trial.Trial + +.. _resources-docstring: + +Resources +--------- + +.. autoclass:: ray.tune.resources.Resources + + + +Registry +-------- + +.. autofunction:: ray.tune.register_trainable + +.. autofunction:: ray.tune.register_env diff --git a/doc/source/tune/api_docs/overview.rst b/doc/source/tune/api_docs/overview.rst new file mode 100644 index 000000000..d2fb60609 --- /dev/null +++ b/doc/source/tune/api_docs/overview.rst @@ -0,0 +1,21 @@ +Tune API Reference +================== + +This section contains a reference for the Tune API. If there is anything missing, please open an issue +on `Github`_. + +.. _`GitHub`: https://github.com/ray-project/ray/issues + +.. toctree:: + :maxdepth: 2 + + execution.rst + trainable.rst + analysis.rst + grid_random.rst + suggestion.rst + schedulers.rst + internals.rst + reporters.rst + client.rst + cli.rst diff --git a/doc/source/tune/api_docs/reporters.rst b/doc/source/tune/api_docs/reporters.rst new file mode 100644 index 000000000..4b784aa67 --- /dev/null +++ b/doc/source/tune/api_docs/reporters.rst @@ -0,0 +1,92 @@ +Console Output (Reporters) +========================== + +By default, Tune reports experiment progress periodically to the command-line as follows. + +.. code-block:: bash + + == Status == + Memory usage on this node: 11.4/16.0 GiB + Using FIFO scheduling algorithm. + Resources requested: 4/12 CPUs, 0/0 GPUs, 0.0/3.17 GiB heap, 0.0/1.07 GiB objects + Result logdir: /Users/foo/ray_results/myexp + Number of trials: 4 (4 RUNNING) + +----------------------+----------+---------------------+-----------+--------+--------+--------+--------+------------------+-------+ + | Trial name | status | loc | param1 | param2 | param3 | acc | loss | total time (s) | iter | + |----------------------+----------+---------------------+-----------+--------+--------+--------+--------+------------------+-------| + | MyTrainable_a826033a | RUNNING | 10.234.98.164:31115 | 0.303706 | 0.0761 | 0.4328 | 0.1289 | 1.8572 | 7.54952 | 15 | + | MyTrainable_a8263fc6 | RUNNING | 10.234.98.164:31117 | 0.929276 | 0.158 | 0.3417 | 0.4865 | 1.6307 | 7.0501 | 14 | + | MyTrainable_a8267914 | RUNNING | 10.234.98.164:31111 | 0.068426 | 0.0319 | 0.1147 | 0.9585 | 1.9603 | 7.0477 | 14 | + | MyTrainable_a826b7bc | RUNNING | 10.234.98.164:31112 | 0.729127 | 0.0748 | 0.1784 | 0.1797 | 1.7161 | 7.05715 | 14 | + +----------------------+----------+---------------------+-----------+--------+--------+--------+--------+------------------+-------+ + +Note that columns will be hidden if they are completely empty. The output can be configured in various ways by instantiating a ``CLIReporter`` instance (or ``JupyterNotebookReporter`` if you're using jupyter notebook). Here's an example: + +.. code-block:: python + + from ray.tune import CLIReporter + + # Limit the number of rows. + reporter = CLIReporter(max_progress_rows=10) + # Add a custom metric column, in addition to the default metrics. + # Note that this must be a metric that is returned in your training results. + reporter.add_metric_column("custom_metric") + tune.run(my_trainable, progress_reporter=reporter) + +Extending ``CLIReporter`` lets you control reporting frequency. For example: + +.. code-block:: python + + class ExperimentTerminationReporter(CLIReporter): + def should_report(self, trials, done=False): + """Reports only on experiment termination.""" + return done + + tune.run(my_trainable, progress_reporter=ExperimentTerminationReporter()) + + class TrialTerminationReporter(CLIReporter): + def __init__(self): + self.num_terminated = 0 + + def should_report(self, trials, done=False): + """Reports only on trial termination events.""" + old_num_terminated = self.num_terminated + self.num_terminated = len([t for t in trials if t.status == Trial.TERMINATED]) + return self.num_terminated > old_num_terminated + + tune.run(my_trainable, progress_reporter=TrialTerminationReporter()) + +The default reporting style can also be overriden more broadly by extending the ``ProgressReporter`` interface directly. Note that you can print to any output stream, file etc. + +.. code-block:: python + + from ray.tune import ProgressReporter + + class CustomReporter(ProgressReporter): + + def should_report(self, trials, done=False): + return True + + def report(self, trials, *sys_info): + print(*sys_info) + print("\n".join([str(trial) for trial in trials])) + + tune.run(my_trainable, progress_reporter=CustomReporter()) + +ProgressReporter +---------------- + +.. autoclass:: ray.tune.ProgressReporter + :members: + +CLIReporter +----------- + +.. autoclass:: ray.tune.CLIReporter + +JupyterNotebookReporter +----------------------- + +.. autoclass:: ray.tune.JupyterNotebookReporter + + diff --git a/doc/source/tune/api_docs/schedulers.rst b/doc/source/tune/api_docs/schedulers.rst new file mode 100644 index 000000000..cc06f51b9 --- /dev/null +++ b/doc/source/tune/api_docs/schedulers.rst @@ -0,0 +1,36 @@ +Schedulers (tune.schedulers) +============================ + +FIFOScheduler +~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.schedulers.FIFOScheduler + +HyperBandScheduler +~~~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.schedulers.HyperBandScheduler + +ASHAScheduler/AsyncHyperBandScheduler +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.schedulers.AsyncHyperBandScheduler + +.. autoclass:: ray.tune.schedulers.ASHAScheduler + +MedianStoppingRule +~~~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.schedulers.MedianStoppingRule + +PopulationBasedTraining +~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.schedulers.PopulationBasedTraining + + +TrialScheduler +~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.schedulers.TrialScheduler + :members: diff --git a/doc/source/tune/api_docs/suggestion.rst b/doc/source/tune/api_docs/suggestion.rst new file mode 100644 index 000000000..45f62d682 --- /dev/null +++ b/doc/source/tune/api_docs/suggestion.rst @@ -0,0 +1,63 @@ +Search Algorithms (tune.suggest) +================================ + +.. _repeater-doc: + +Repeater +-------- + +.. autoclass:: ray.tune.suggest.Repeater + +AxSearch +-------- + +.. autoclass:: ray.tune.suggest.ax.AxSearch + +BayesOptSearch +-------------- + +.. autoclass:: ray.tune.suggest.bayesopt.BayesOptSearch + +TuneBOHB +-------- + +.. autoclass:: ray.tune.suggest.bohb.TuneBOHB + +DragonflySearch +--------------- + +.. autoclass:: ray.tune.suggest.dragonfly.DragonflySearch + +HyperOptSearch +-------------- + +.. autoclass:: ray.tune.suggest.hyperopt.HyperOptSearch + +NevergradSearch +--------------- + +.. autoclass:: ray.tune.suggest.nevergrad.NevergradSearch + +SigOptSearch +------------ + +.. autoclass:: ray.tune.suggest.sigopt.SigOptSearch + +SkOptSearch +----------- + +.. autoclass:: ray.tune.suggest.skopt.SkOptSearch + +SearchAlgorithm +--------------- + +.. autoclass:: ray.tune.suggest.SearchAlgorithm + :members: + +SuggestionAlgorithm +------------------- + +.. autoclass:: ray.tune.suggest.SuggestionAlgorithm + :members: + :private-members: + :show-inheritance: diff --git a/doc/source/tune/api_docs/trainable.rst b/doc/source/tune/api_docs/trainable.rst new file mode 100644 index 000000000..92bdd6073 --- /dev/null +++ b/doc/source/tune/api_docs/trainable.rst @@ -0,0 +1,32 @@ +Training (tune.Trainable, tune.track) +===================================== + +.. _trainable-docstring: + +tune.Trainable +~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.Trainable + :member-order: groupwise + :private-members: + :members: + +tune.DurableTrainable +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.DurableTrainable + +.. _track-docstring: + +tune.track +~~~~~~~~~~ + +.. automodule:: ray.tune.track + :members: + :exclude-members: init, shutdown + +StatusReporter +~~~~~~~~~~~~~~ + +.. autoclass:: ray.tune.function_runner.StatusReporter + :members: __call__, logdir