diff --git a/ci/jenkins_tests/run_tune_tests.sh b/ci/jenkins_tests/run_tune_tests.sh index 6931aa900..326cf5543 100755 --- a/ci/jenkins_tests/run_tune_tests.sh +++ b/ci/jenkins_tests/run_tune_tests.sh @@ -70,6 +70,12 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} python /ray/python/ray/tune/examples/tune_mnist_async_hyperband.py \ --smoke-test +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + python /ray/python/ray/tune/examples/lightgbm_example.py + +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + python /ray/python/ray/tune/examples/xgboost_example.py + $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ python /ray/python/ray/tune/examples/logging_example.py \ --smoke-test diff --git a/doc/source/tune-examples.rst b/doc/source/tune-examples.rst index a15f5ec4e..cb0a4c9d0 100644 --- a/doc/source/tune-examples.rst +++ b/doc/source/tune-examples.rst @@ -11,54 +11,56 @@ If any example is broken, or if you'd like to add an example to this page, feel General Examples ---------------- -- `async_hyperband_example `__: - Example of using a Trainable class with AsyncHyperBandScheduler. -- `hyperband_example `__: - Example of using a Trainable class with HyperBandScheduler. -- `hyperopt_example `__: - Optimizes a basic function using the function-based API and the HyperOptSearch (SearchAlgorithm wrapper for HyperOpt TPE). - Also uses the AsyncHyperBandScheduler. -- `pbt_example `__: - Example of using a Trainable class with PopulationBasedTraining scheduler. -- `pbt_ppo_example `__: - Example of optimizing a distributed RLlib algorithm (PPO) with the PopulationBasedTraining scheduler. -- `logging_example `__: - Example of custom loggers and custom trial directory naming. +- `async_hyperband_example `__: Example of using a Trainable class with AsyncHyperBandScheduler. +- `hyperband_example `__: Example of using a Trainable class with HyperBandScheduler. Also uses the Experiment class API for specifying the experiment configuration. Also uses the AsyncHyperBandScheduler. +- `pbt_example `__: Example of using a Trainable class with PopulationBasedTraining scheduler. +- `pbt_ppo_example `__: Example of optimizing a distributed RLlib algorithm (PPO) with the PopulationBasedTraining scheduler. +- `logging_example `__: Example of custom loggers and custom trial directory naming. +Search Algorithm Examples +------------------------- + +- `Ax example `__: Optimize a Hartmann function with `Ax `_ with 4 parallel workers. +- `HyperOpt Example `__: Optimizes a basic function using the function-based API and the HyperOptSearch (SearchAlgorithm wrapper for HyperOpt TPE). +- `Nevergrad example `__: Optimize a simple toy function with the gradient-free optimization package `Nevergrad `_ with 4 parallel workers. +- `Bayesian Optimization example `__: Optimize a simple toy function using `Bayesian Optimization `_ with 4 parallel workers. Keras Examples -------------- -- `tune_mnist_keras `__: - Converts the Keras MNIST example to use Tune with the function-based API and a Keras callback. +- `tune_mnist_keras `__: Converts the Keras MNIST example to use Tune with the function-based API and a Keras callback. Also shows how to easily convert something relying on argparse to use Tune. PyTorch Examples ---------------- -- `mnist_pytorch `__: - Converts the PyTorch MNIST example to use Tune with the function-based API. Also shows how to easily convert something relying on argparse to use Tune. -- `mnist_pytorch_trainable `__: - Converts the PyTorch MNIST example to use Tune with Trainable API. Also uses the HyperBandScheduler and checkpoints the model at the end. +- `mnist_pytorch `__: Converts the PyTorch MNIST example to use Tune with the function-based API. Also shows how to easily convert something relying on argparse to use Tune. +- `mnist_pytorch_trainable `__: Converts the PyTorch MNIST example to use Tune with Trainable API. Also uses the HyperBandScheduler and checkpoints the model at the end. TensorFlow Examples ------------------- -- `tune_mnist_ray `__: - A basic example of tuning a TensorFlow model on MNIST using the Trainable class. -- `tune_mnist_ray_hyperband `__: - A basic example of tuning a TensorFlow model on MNIST using the Trainable class and the HyperBand scheduler. -- `tune_mnist_async_hyperband `__: - Example of tuning a TensorFlow model on MNIST using AsyncHyperBand. +- `tune_mnist_ray `__: A basic example of tuning a TensorFlow model on MNIST using the Trainable class. +- `tune_mnist_ray_hyperband `__: A basic example of tuning a TensorFlow model on MNIST using the Trainable class and the HyperBand scheduler. +- `tune_mnist_async_hyperband `__: Example of tuning a TensorFlow model on MNIST using AsyncHyperBand. + + +XGBoost Example +--------------- + +- `xgboost_example `__: Trains a basic XGBoost model with Tune with the function-based API and a XGBoost callback. + + +LightGBM Example +---------------- + +- `lightgbm_example `__: Trains a basic LightGBM model with Tune with the function-based API and a LightGBM callback. Contributed Examples -------------------- -- `pbt_tune_cifar10_with_keras `__: - A contributed example of tuning a Keras model on CIFAR10 with the PopulationBasedTraining scheduler. -- `genetic_example `__: - Optimizing the michalewicz function using the contributed GeneticSearch search algorithm with AsyncHyperBandScheduler. - - +- `pbt_tune_cifar10_with_keras `__: A contributed example of tuning a Keras model on CIFAR10 with the PopulationBasedTraining scheduler. +- `genetic_example `__: Optimizing the michalewicz function using the contributed GeneticSearch search algorithm with AsyncHyperBandScheduler. +- `tune_cifar10_gluon `__: MXNet Gluon example to use Tune with the function-based API on CIFAR-10 dataset. diff --git a/docker/examples/Dockerfile b/docker/examples/Dockerfile index a4358eec7..4767ca360 100644 --- a/docker/examples/Dockerfile +++ b/docker/examples/Dockerfile @@ -11,7 +11,7 @@ RUN pip install gym[atari] opencv-python-headless tensorflow lz4 keras pytest-ti RUN pip install -U h5py # Mutes FutureWarnings RUN pip install --upgrade bayesian-optimization RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git -RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster ConfigSpace +RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster ConfigSpace lightgbm xgboost RUN pip install -U mlflow RUN pip install -U pytest-remotedata>=0.3.1 RUN conda install pytorch-cpu torchvision-cpu -c pytorch diff --git a/docker/tune_test/Dockerfile b/docker/tune_test/Dockerfile index dbd12ed8b..a2d727a79 100644 --- a/docker/tune_test/Dockerfile +++ b/docker/tune_test/Dockerfile @@ -13,7 +13,7 @@ RUN conda remove -y --force wrapt RUN pip install gym[atari]==0.10.11 opencv-python-headless tensorflow lz4 keras pytest-timeout smart_open RUN pip install --upgrade bayesian-optimization RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git -RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster ConfigSpace +RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster ConfigSpace lightgbm xgboost RUN pip install -U mlflow RUN pip install -U pytest-remotedata>=0.3.1 RUN conda install pytorch-cpu torchvision-cpu -c pytorch diff --git a/python/ray/tune/__init__.py b/python/ray/tune/__init__.py index 6c45df1e1..b5ab4fb80 100644 --- a/python/ray/tune/__init__.py +++ b/python/ray/tune/__init__.py @@ -10,11 +10,11 @@ from ray.tune.registry import register_env, register_trainable from ray.tune.trainable import Trainable from ray.tune.suggest import grid_search from ray.tune.sample import (function, sample_from, uniform, choice, randint, - randn) + randn, loguniform) __all__ = [ "Trainable", "TuneError", "grid_search", "register_env", "register_trainable", "run", "run_experiments", "Experiment", "function", "sample_from", "track", "uniform", "choice", "randint", "randn", - "ExperimentAnalysis", "Analysis" + "loguniform", "ExperimentAnalysis", "Analysis" ] diff --git a/python/ray/tune/examples/README.rst b/python/ray/tune/examples/README.rst index 79fef2c05..d2f48cba0 100644 --- a/python/ray/tune/examples/README.rst +++ b/python/ray/tune/examples/README.rst @@ -11,54 +11,56 @@ If any example is broken, or if you'd like to add an example to this page, feel General Examples ---------------- -- `async_hyperband_example `__: - Example of using a Trainable class with AsyncHyperBandScheduler. -- `hyperband_example `__: - Example of using a Trainable class with HyperBandScheduler. Also uses the Experiment class API for specifying the experiment configuration. -- `hyperopt_example `__: - Optimizes a basic function using the function-based API and the HyperOptSearch (SearchAlgorithm wrapper for HyperOpt TPE). - Also uses the AsyncHyperBandScheduler. -- `pbt_example `__: - Example of using a Trainable class with PopulationBasedTraining scheduler. -- `pbt_ppo_example `__: - Example of optimizing a distributed RLlib algorithm (PPO) with the PopulationBasedTraining scheduler. -- `logging_example `__: - Example of custom loggers and custom trial directory naming. +- `async_hyperband_example `__: Example of using a Trainable class with AsyncHyperBandScheduler. +- `hyperband_example `__: Example of using a Trainable class with HyperBandScheduler. Also uses the Experiment class API for specifying the experiment configuration. Also uses the AsyncHyperBandScheduler. +- `pbt_example `__: Example of using a Trainable class with PopulationBasedTraining scheduler. +- `pbt_ppo_example `__: Example of optimizing a distributed RLlib algorithm (PPO) with the PopulationBasedTraining scheduler. +- `logging_example `__: Example of custom loggers and custom trial directory naming. +Search Algorithm Examples +------------------------- + +- `Ax example `__: Optimize a Hartmann function with `Ax `_ with 4 parallel workers. +- `HyperOpt Example `__: Optimizes a basic function using the function-based API and the HyperOptSearch (SearchAlgorithm wrapper for HyperOpt TPE). +- `Nevergrad example `__: Optimize a simple toy function with the gradient-free optimization package `Nevergrad `_ with 4 parallel workers. +- `Bayesian Optimization example `__: Optimize a simple toy function using `Bayesian Optimization `_ with 4 parallel workers. Keras Examples -------------- -- `tune_mnist_keras `__: - Converts the Keras MNIST example to use Tune with the function-based API and a Keras callback. Also shows how to easily convert something relying on argparse to use Tune. +- `tune_mnist_keras `__: Converts the Keras MNIST example to use Tune with the function-based API and a Keras callback. Also shows how to easily convert something relying on argparse to use Tune. PyTorch Examples ---------------- -- `mnist_pytorch `__: - Converts the PyTorch MNIST example to use Tune with the function-based API. Also shows how to easily convert something relying on argparse to use Tune. -- `mnist_pytorch_trainable `__: - Converts the PyTorch MNIST example to use Tune with Trainable API. Also uses the HyperBandScheduler and checkpoints the model at the end. +- `mnist_pytorch `__: Converts the PyTorch MNIST example to use Tune with the function-based API. Also shows how to easily convert something relying on argparse to use Tune. +- `mnist_pytorch_trainable `__: Converts the PyTorch MNIST example to use Tune with Trainable API. Also uses the HyperBandScheduler and checkpoints the model at the end. TensorFlow Examples ------------------- -- `tune_mnist_ray `__: - A basic example of tuning a TensorFlow model on MNIST using the Trainable class. -- `tune_mnist_ray_hyperband `__: - A basic example of tuning a TensorFlow model on MNIST using the Trainable class and the HyperBand scheduler. -- `tune_mnist_async_hyperband `__: - Example of tuning a TensorFlow model on MNIST using AsyncHyperBand. +- `tune_mnist_ray `__: A basic example of tuning a TensorFlow model on MNIST using the Trainable class. +- `tune_mnist_ray_hyperband `__: A basic example of tuning a TensorFlow model on MNIST using the Trainable class and the HyperBand scheduler. +- `tune_mnist_async_hyperband `__: Example of tuning a TensorFlow model on MNIST using AsyncHyperBand. + + +XGBoost Example +--------------- + +- `xgboost_example `__: Trains a basic XGBoost model with Tune with the function-based API and a XGBoost callback. + + +LightGBM Example +---------------- + +- `lightgbm_example `__: Trains a basic LightGBM model with Tune with the function-based API and a LightGBM callback. Contributed Examples -------------------- -- `pbt_tune_cifar10_with_keras `__: - A contributed example of tuning a Keras model on CIFAR10 with the PopulationBasedTraining scheduler. -- `genetic_example `__: - Optimizing the michalewicz function using the contributed GeneticSearch search algorithm with AsyncHyperBandScheduler. -- `tune_cifar10_gluon `__: - MXNet Gluon example to use Tune with the function-based API on CIFAR-10 dataset. +- `pbt_tune_cifar10_with_keras `__: A contributed example of tuning a Keras model on CIFAR10 with the PopulationBasedTraining scheduler. +- `genetic_example `__: Optimizing the michalewicz function using the contributed GeneticSearch search algorithm with AsyncHyperBandScheduler. +- `tune_cifar10_gluon `__: MXNet Gluon example to use Tune with the function-based API on CIFAR-10 dataset. diff --git a/python/ray/tune/examples/lightgbm_example.py b/python/ray/tune/examples/lightgbm_example.py new file mode 100644 index 000000000..7a45bb3ce --- /dev/null +++ b/python/ray/tune/examples/lightgbm_example.py @@ -0,0 +1,53 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import lightgbm as lgb +import numpy as np +import sklearn.datasets +import sklearn.metrics +from sklearn.model_selection import train_test_split + +from ray import tune + + +def LightGBMCallback(env): + """Assumes that `valid_0` is the target validation score.""" + _, metric, score, _ = env.evaluation_result_list[0] + tune.track.log(**{metric: score}) + + +def train_breast_cancer(config): + data, target = sklearn.datasets.load_breast_cancer(return_X_y=True) + train_x, test_x, train_y, test_y = train_test_split( + data, target, test_size=0.25) + train_set = lgb.Dataset(train_x, label=train_y) + test_set = lgb.Dataset(test_x, label=test_y) + gbm = lgb.train( + config, + train_set, + valid_sets=[test_set], + verbose_eval=False, + callbacks=[LightGBMCallback]) + preds = gbm.predict(test_x) + pred_labels = np.rint(preds) + tune.track.log( + mean_accuracy=sklearn.metrics.accuracy_score(test_y, pred_labels), + done=True) + + +if __name__ == "__main__": + config = { + "objective": "binary", + "metric": "binary_error", + "verbose": -1, + "boosting_type": tune.grid_search(["gbdt", "dart"]), + "num_leaves": tune.randint(10, 1000), + "learning_rate": tune.loguniform(1e-8, 1e-1) + } + from ray.tune.schedulers import ASHAScheduler + tune.run( + train_breast_cancer, + config=config, + num_samples=2, + scheduler=ASHAScheduler(metric="binary_error", mode="min")) diff --git a/python/ray/tune/examples/xgboost_example.py b/python/ray/tune/examples/xgboost_example.py new file mode 100644 index 000000000..ca295de07 --- /dev/null +++ b/python/ray/tune/examples/xgboost_example.py @@ -0,0 +1,53 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import xgboost as xgb +import numpy as np +import sklearn.datasets +import sklearn.metrics +from sklearn.model_selection import train_test_split + +from ray import tune + + +def XGBCallback(env): + tune.track.log(**dict(env.evaluation_result_list)) + + +def train_breast_cancer(config): + data, target = sklearn.datasets.load_breast_cancer(return_X_y=True) + train_x, test_x, train_y, test_y = train_test_split( + data, target, test_size=0.25) + train_set = xgb.DMatrix(train_x, label=train_y) + test_set = xgb.DMatrix(test_x, label=test_y) + bst = xgb.train( + config, train_set, evals=[(test_set, "eval")], callbacks=[XGBCallback]) + preds = bst.predict(test_set) + pred_labels = np.rint(preds) + tune.track.log( + mean_accuracy=sklearn.metrics.accuracy_score(test_y, pred_labels), + done=True) + + +if __name__ == "__main__": + num_threads = 2 + config = { + "verbosity": 0, + "num_threads": num_threads, + "objective": "binary:logistic", + "booster": "gbtree", + "eval_metric": ["auc", "ams@0", "logloss"], + "max_depth": tune.randint(1, 9), + "eta": tune.loguniform(1e-4, 1e-1), + "gamma": tune.loguniform(1e-8, 1.0), + "grow_policy": tune.choice(["depthwise", "lossguide"]) + } + + from ray.tune.schedulers import ASHAScheduler + tune.run( + train_breast_cancer, + resources_per_trial={"cpu": num_threads}, + config=config, + num_samples=2, + scheduler=ASHAScheduler(metric="eval-logloss", mode="min")) diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py index d40990a27..f06470f62 100644 --- a/python/ray/tune/sample.py +++ b/python/ray/tune/sample.py @@ -56,6 +56,23 @@ def uniform(*args, **kwargs): return sample_from(lambda _: np.random.uniform(*args, **kwargs)) +def loguniform(min_bound, max_bound, base=10): + """Sugar for sampling in different orders of magnitude. + + Args: + min_bound (float): Lower boundary of the output interval (1e-4) + max_bound (float): Upper boundary of the output interval (1e-2) + base (float): Base of the log. Defaults to 10. + """ + logmin = np.log(min_bound) / np.log(base) + logmax = np.log(max_bound) / np.log(base) + + def apply_log(_): + return base**(np.random.uniform(logmin, logmax)) + + return sample_from(apply_log) + + def choice(*args, **kwargs): """A wrapper around np.random.choice.""" return sample_from(lambda _: np.random.choice(*args, **kwargs)) diff --git a/python/ray/tune/schedulers/async_hyperband.py b/python/ray/tune/schedulers/async_hyperband.py index d3e56d305..89ce09669 100644 --- a/python/ray/tune/schedulers/async_hyperband.py +++ b/python/ray/tune/schedulers/async_hyperband.py @@ -92,6 +92,8 @@ class AsyncHyperBandScheduler(FIFOScheduler): def on_trial_result(self, trial_runner, trial, result): action = TrialScheduler.CONTINUE + if self._time_attr not in result or self._metric not in result: + return action if result[self._time_attr] >= self._max_t: action = TrialScheduler.STOP else: @@ -103,6 +105,8 @@ class AsyncHyperBandScheduler(FIFOScheduler): return action def on_trial_complete(self, trial_runner, trial, result): + if self._time_attr not in result or self._metric not in result: + return bracket = self._trial_info[trial.trial_id] bracket.on_result(trial, result[self._time_attr], self._metric_op * result[self._metric]) diff --git a/python/ray/tune/schedulers/median_stopping_rule.py b/python/ray/tune/schedulers/median_stopping_rule.py index 36276273c..3609eee73 100644 --- a/python/ray/tune/schedulers/median_stopping_rule.py +++ b/python/ray/tune/schedulers/median_stopping_rule.py @@ -77,6 +77,8 @@ class MedianStoppingRule(FIFOScheduler): value by step `t` is strictly worse than the median of the running averages of all completed trials' objectives reported up to step `t`. """ + if self._time_attr not in result or self._metric not in result: + return TrialScheduler.CONTINUE if trial in self._stopped_trials: assert not self._hard_stop diff --git a/python/ray/tune/schedulers/pbt.py b/python/ray/tune/schedulers/pbt.py index 2953c1327..26fa5f6d9 100644 --- a/python/ray/tune/schedulers/pbt.py +++ b/python/ray/tune/schedulers/pbt.py @@ -219,6 +219,8 @@ class PopulationBasedTraining(FIFOScheduler): self._trial_state[trial] = PBTTrialState(trial) def on_trial_result(self, trial_runner, trial, result): + if self._time_attr not in result or self._metric not in result: + return TrialScheduler.CONTINUE time = result[self._time_attr] state = self._trial_state[trial] diff --git a/python/ray/tune/tests/test_trial_runner.py b/python/ray/tune/tests/test_trial_runner.py index 7f9df19db..c53bd84de 100644 --- a/python/ray/tune/tests/test_trial_runner.py +++ b/python/ray/tune/tests/test_trial_runner.py @@ -5,6 +5,7 @@ from __future__ import print_function import copy import glob import os +import numpy as np import shutil import sys import tempfile @@ -1298,6 +1299,17 @@ class VariantGeneratorTest(unittest.TestCase): self.assertEqual(trials[0].config, {"x": 100, "y": 1}) self.assertEqual(trials[1].config, {"x": 200, "y": 1}) + def testLogUniform(self): + sampler = tune.loguniform(1e-10, 1e-1).func + results = [sampler(None) for i in range(1000)] + assert abs(np.log(min(results)) / np.log(10) - -10) < 0.1 + assert abs(np.log(max(results)) / np.log(10) - -1) < 0.1 + + sampler_e = tune.loguniform(np.e**-4, np.e, base=np.e).func + results_e = [sampler_e(None) for i in range(1000)] + assert abs(np.log(min(results_e)) - -4) < 0.1 + assert abs(np.log(max(results_e)) - 1) < 0.1 + def test_resolve_dict(self): config = { "a": {