diff --git a/.travis.yml b/.travis.yml index 2856d6839..045903c44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -247,6 +247,82 @@ matrix: script: - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/... + + # Tune: Tests and examples. + - os: linux + env: + - TUNE_TESTING=1 + - PYTHON=3.6 + - TF_VERSION=2.1.0 + - TFP_VERSION=0.8 + - TORCH_VERSION=1.5 + - PYTHONWARNINGS=ignore + install: + - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED + before_script: + - . ./ci/travis/ci.sh build + script: + - ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only,-example python/ray/tune/... + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=example,-tf,-pytorch,-py37,-flaky python/ray/tune/... + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-flaky python/ray/tune/... + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-flaky python/ray/tune/... + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-py37,flaky python/ray/tune/... + + # SGD: Tests and examples. + - os: linux + env: + - SGD_TESTING=1 + - PYTHON=3.6 + - TF_VERSION=2.1.0 + - TFP_VERSION=0.8 + - TORCH_VERSION=1.5 + - PYTHONWARNINGS=ignore + install: + - . ./ci/travis/ci.sh init RAY_CI_SGD_AFFECTED + before_script: + - . ./ci/travis/ci.sh build + script: + # No such tests at the moment: + # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,-pytorch,-py37 python/ray/util/sgd/... + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tf,-pytorch,-py37 python/ray/util/sgd/... + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,pytorch,-py37 python/ray/util/sgd/... + + # Docs: Tests and examples. + - os: linux + env: + - DOC_TESTING=1 + - PYTHON=3.6 + - TF_VERSION=2.1.0 + - TFP_VERSION=0.8 + - TORCH_VERSION=1.5 + - PYTHONWARNINGS=ignore + install: + - . ./ci/travis/ci.sh init RAY_CI_PYTHON_AFFECTED + before_script: + - . ./ci/travis/ci.sh build + script: + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,-pytorch,-py37 doc/... + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tf,-pytorch,-py37 doc/... + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,pytorch,-py37 doc/... + + # Tune/SGD/Docs: Python >= 3.7 tests and examples. + - os: linux + env: + - PYTHON=3.7 TUNE_TESTING=1 + - TF_VERSION=2.1.0 + - TFP_VERSION=0.8 + - TORCH_VERSION=1.5 + - PYTHONWARNINGS=ignore + install: + - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED,RAY_CI_SGD_AFFECTED + before_script: + - . ./ci/travis/ci.sh build + script: + - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 python/ray/tune/... + # There are no python 3.7 tests for RaySGD/Docs at the moment + # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 python/ray/util/sgd/... + # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 doc/... + # Cpp worker test - os: linux env: @@ -281,7 +357,6 @@ script: # bazel python tests. This should be run last to keep its logs at the end of travis logs. - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/...; fi - - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tune/...; fi # NO MORE TESTS BELOW, keep them above. deploy: diff --git a/ci/travis/determine_tests_to_run.py b/ci/travis/determine_tests_to_run.py index a792c1abf..5b27f4631 100644 --- a/ci/travis/determine_tests_to_run.py +++ b/ci/travis/determine_tests_to_run.py @@ -33,6 +33,7 @@ def list_changed_files(commit_range): if __name__ == "__main__": RAY_CI_TUNE_AFFECTED = 0 + RAY_CI_SGD_AFFECTED = 0 RAY_CI_ONLY_RLLIB_AFFECTED = 0 # Whether only RLlib is affected. RAY_CI_RLLIB_AFFECTED = 0 # Whether RLlib minimal tests should be run. RAY_CI_RLLIB_FULL_AFFECTED = 0 # Whether full RLlib tests should be run. @@ -73,6 +74,10 @@ if __name__ == "__main__": RAY_CI_RLLIB_FULL_AFFECTED = 1 RAY_CI_LINUX_WHEELS_AFFECTED = 1 RAY_CI_MACOS_WHEELS_AFFECTED = 1 + elif changed_file.startswith("python/ray/util/sgd"): + RAY_CI_SGD_AFFECTED = 1 + RAY_CI_LINUX_WHEELS_AFFECTED = 1 + RAY_CI_MACOS_WHEELS_AFFECTED = 1 elif re.match("^(python/ray/)?rllib/", changed_file): RAY_CI_RLLIB_AFFECTED = 1 RAY_CI_RLLIB_FULL_AFFECTED = 1 @@ -86,6 +91,7 @@ if __name__ == "__main__": RAY_CI_DASHBOARD_AFFECTED = 1 elif changed_file.startswith("python/"): RAY_CI_TUNE_AFFECTED = 1 + RAY_CI_SGD_AFFECTED = 1 RAY_CI_RLLIB_AFFECTED = 1 RAY_CI_SERVE_AFFECTED = 1 RAY_CI_PYTHON_AFFECTED = 1 @@ -103,6 +109,7 @@ if __name__ == "__main__": pass elif changed_file.startswith("src/"): RAY_CI_TUNE_AFFECTED = 1 + RAY_CI_SGD_AFFECTED = 1 RAY_CI_RLLIB_AFFECTED = 1 RAY_CI_SERVE_AFFECTED = 1 RAY_CI_JAVA_AFFECTED = 1 @@ -123,6 +130,7 @@ if __name__ == "__main__": RAY_CI_STREAMING_JAVA_AFFECTED = 1 else: RAY_CI_TUNE_AFFECTED = 1 + RAY_CI_SGD_AFFECTED = 1 RAY_CI_RLLIB_AFFECTED = 1 RAY_CI_SERVE_AFFECTED = 1 RAY_CI_JAVA_AFFECTED = 1 @@ -134,6 +142,7 @@ if __name__ == "__main__": RAY_CI_STREAMING_JAVA_AFFECTED = 1 else: RAY_CI_TUNE_AFFECTED = 1 + RAY_CI_SGD_AFFECTED = 1 RAY_CI_RLLIB_AFFECTED = 1 RAY_CI_RLLIB_FULL_AFFECTED = 1 RAY_CI_SERVE_AFFECTED = 1 @@ -149,12 +158,14 @@ if __name__ == "__main__": not RAY_CI_JAVA_AFFECTED and not RAY_CI_PYTHON_AFFECTED and not \ RAY_CI_STREAMING_CPP_AFFECTED and \ not RAY_CI_STREAMING_PYTHON_AFFECTED and \ - not RAY_CI_STREAMING_JAVA_AFFECTED: + not RAY_CI_STREAMING_JAVA_AFFECTED and \ + not RAY_CI_SGD_AFFECTED: RAY_CI_ONLY_RLLIB_AFFECTED = 1 # Log the modified environment variables visible in console. print(" ".join([ "RAY_CI_TUNE_AFFECTED={}".format(RAY_CI_TUNE_AFFECTED), + "RAY_CI_SGD_AFFECTED={}".format(RAY_CI_SGD_AFFECTED), "RAY_CI_ONLY_RLLIB_AFFECTED={}".format(RAY_CI_ONLY_RLLIB_AFFECTED), "RAY_CI_RLLIB_AFFECTED={}".format(RAY_CI_RLLIB_AFFECTED), "RAY_CI_RLLIB_FULL_AFFECTED={}".format(RAY_CI_RLLIB_FULL_AFFECTED), diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 1301c7a8a..bf6f16829 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -279,6 +279,27 @@ install_dependencies() { torch=="${TORCH_VERSION-1.4}" torchvision atari_py "gym[atari]" lz4 smart_open fi + # Additional Tune test dependencies. + if [ "${TUNE_TESTING-}" = 1 ]; then + pip install tensorflow-probability=="${TFP_VERSION-0.8}" \ + torch=="${TORCH_VERSION-1.4}" + pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt + fi + + # Additional RaySGD test dependencies. + if [ "${SGD_TESTING-}" = 1 ]; then + pip install tensorflow-probability=="${TFP_VERSION-0.8}" \ + torch=="${TORCH_VERSION-1.4}" + pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt + fi + + # Additional Doc test dependencies. + if [ "${DOC_TESTING-}" = 1 ]; then + pip install tensorflow-probability=="${TFP_VERSION-0.8}" \ + torch=="${TORCH_VERSION-1.4}" torchvision atari_py gym[atari] lz4 smart_open + pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt + fi + # Additional streaming dependencies. if [ "${RAY_CI_STREAMING_PYTHON_AFFECTED}" = 1 ]; then pip install "msgpack>=1.0.0" diff --git a/doc/BUILD b/doc/BUILD new file mode 100644 index 000000000..3ad15c58f --- /dev/null +++ b/doc/BUILD @@ -0,0 +1,56 @@ +# -------------------------------------------------------------------- +# Tests from the doc/examples directory. +# Please keep these sorted alphabetically, but start with the +# root directory. +# -------------------------------------------------------------------- +py_test( + name = "plot_hyperparameter", + size = "small", + srcs = ["examples/plot_hyperparameter.py"], + tags = ["exclusive"] +) + +py_test( + name = "plot_parameter_server", + size = "medium", + srcs = ["examples/plot_parameter_server.py"], + tags = ["exclusive"] +) + +py_test( + name = "plot_pong_example", + size = "large", + srcs = ["examples/plot_pong_example.py"], + tags = ["exclusive"] +) + +# Directory: examples/doc_code +py_test( + name = "doc_code_tf_example", + size = "small", + main = "examples/doc_code/tf_example.py", + srcs = ["examples/doc_code/tf_example.py"], + tags = ["exclusive", "tf"] +) + +py_test( + name = "doc_code_torch_example", + size = "small", + main = "examples/doc_code/torch_example.py", + srcs = ["examples/doc_code/torch_example.py"], + tags = ["exclusive", "pytorch"] +) + +# -------------------------------------------------------------------- +# Tests from the doc/source/tune/_tutorials directory. +# Please keep these sorted alphabetically. +# -------------------------------------------------------------------- + +py_test( + name = "tune_sklearn", + size = "medium", + main = "source/tune/_tutorials/tune-sklearn.py", + srcs = ["source/tune/_tutorials/tune-sklearn.py"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) diff --git a/docker/tune_test/requirements.txt b/docker/tune_test/requirements.txt index 58294f5ce..a6077e323 100644 --- a/docker/tune_test/requirements.txt +++ b/docker/tune_test/requirements.txt @@ -1,8 +1,10 @@ +ax-platform ConfigSpace==0.4.10 bayesian-optimization boto3 dm_tree dragonfly-opt +gluoncv gym[atari] h5py hpbandster @@ -13,6 +15,7 @@ lightgbm lz4 matplotlib mlflow +mxnet nevergrad opencv-python-headless pandas @@ -26,8 +29,8 @@ smart_open tabulate tensorboardX tensorflow_probability -torch -torchvision +torch>=1.5.0 +torchvision>=0.6.0 xgboost zoopt>=0.4.0 timm diff --git a/python/ray/tune/BUILD b/python/ray/tune/BUILD index 87016914c..fa40e3c70 100644 --- a/python/ray/tune/BUILD +++ b/python/ray/tune/BUILD @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------- +# Tests from the python/ray/tune/tests directory. +# Covers all tests starting with `test_`. +# Please keep these sorted alphabetically. +# -------------------------------------------------------------------- py_test( name = "test_actor_reuse", size = "medium", @@ -6,6 +11,14 @@ py_test( deps = [":tune_lib"], ) +py_test( + name = "test_api", + size = "large", + srcs = ["tests/test_api.py"], + deps = [":tune_lib"], + tags = ["exclusive"], +) + py_test( name = "test_automl_searcher", size = "small", @@ -25,7 +38,7 @@ py_test( size = "large", srcs = ["tests/test_cluster.py"], deps = [":tune_lib"], - tags = ["jenkins_only", "exclusive"], + tags = ["flaky", "jenkins_only", "exclusive"], ) py_test( @@ -51,6 +64,13 @@ py_test( deps = [":tune_lib"], ) +py_test( + name = "test_experiment", + size = "small", + srcs = ["tests/test_experiment.py"], + deps = [":tune_lib"], +) + py_test( name = "test_experiment_analysis", size = "medium", @@ -66,10 +86,11 @@ py_test( ) py_test( - name = "test_experiment", - size = "small", - srcs = ["tests/test_experiment.py"], + name = "test_function_api", + size = "medium", + srcs = ["tests/test_function_api.py"], deps = [":tune_lib"], + tags = ["exclusive"], ) py_test( @@ -94,6 +115,22 @@ py_test( deps = [":tune_lib"], ) +py_test( + name = "test_run_experiment", + size = "medium", + srcs = ["tests/test_run_experiment.py"], + deps = [":tune_lib"], + tags = ["exclusive"], +) + +py_test( + name = "test_sync", + size = "medium", + srcs = ["tests/test_sync.py"], + deps = [":tune_lib"], + tags = ["exclusive"], +) + py_test( name = "test_track", size = "small", @@ -102,11 +139,10 @@ py_test( ) py_test( - name = "test_run_experiment", - size = "medium", - srcs = ["tests/test_run_experiment.py"], + name = "test_trainable_util", + size = "small", + srcs = ["tests/test_trainable_util.py"], deps = [":tune_lib"], - tags = ["exclusive"], ) py_test( @@ -141,37 +177,6 @@ py_test( tags = ["exclusive"], ) -py_test( - name = "test_api", - size = "large", - srcs = ["tests/test_api.py"], - deps = [":tune_lib"], - tags = ["exclusive"], -) - -py_test( - name = "test_function_api", - size = "medium", - srcs = ["tests/test_function_api.py"], - deps = [":tune_lib"], - tags = ["exclusive"], -) - -py_test( - name = "test_sync", - size = "medium", - srcs = ["tests/test_sync.py"], - deps = [":tune_lib"], - tags = ["exclusive"], -) - -py_test( - name = "test_trainable_util", - size = "small", - srcs = ["tests/test_trainable_util.py"], - deps = [":tune_lib"], -) - py_test( name = "test_trial_scheduler", size = "medium", @@ -212,6 +217,312 @@ py_test( tags = ["exclusive"], ) +# -------------------------------------------------------------------- +# Tests from the python/ray/tune/tests directory. +# Covers all remaining tests that do not start with `test_`. +# Please keep these sorted alphabetically. +# -------------------------------------------------------------------- +py_test( + name = "example", + size = "small", + srcs = ["tests/example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], +) + +# Todo: Ensure MPLBACKEND=Agg +py_test( + name = "tutorial", + size = "medium", + srcs = ["tests/tutorial.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], +) + +# -------------------------------------------------------------------- +# Examples from the python/ray/tune/examples directory. +# Please keep these sorted alphabetically. +# -------------------------------------------------------------------- +py_test( + name = "async_hyperband_example", + size = "small", + srcs = ["examples/async_hyperband_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "ax_example", + size = "medium", + srcs = ["examples/ax_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example", "py37"], + args = ["--smoke-test"] +) + +py_test( + name = "bayesopt_example", + size = "medium", + srcs = ["examples/bayesopt_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "bohb_example", + size = "small", + srcs = ["examples/bohb_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"] +) + +py_test( + name = "cifar10_pytorch", + size = "medium", + srcs = ["examples/cifar10_pytorch.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example", "pytorch"], + args = ["--smoke-test"] +) + +py_test( + name = "dragonfly_example", + size = "medium", + srcs = ["examples/dragonfly_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +# Does not work without awscli tool installed. Check if can be mocked. +# py_test( +# name = "durable_trainable_example", +# size = "medium", +# srcs = ["examples/durable_trainable_example.py"], +# deps = [":tune_lib"], +# tags = ["exclusive", "example"], +# args = ["--local", "--mock-storage"] +# ) + +py_test( + name = "genetic_example", + size = "small", + srcs = ["examples/genetic_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "hyperband_example", + size = "medium", + srcs = ["examples/hyperband_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"] +) + +py_test( + name = "hyperband_function_example", + size = "medium", + srcs = ["examples/hyperband_function_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "hyperopt_example", + size = "medium", + srcs = ["examples/hyperopt_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "lightgbm_example", + size = "medium", + srcs = ["examples/lightgbm_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"] +) + +py_test( + name = "logging_example", + size = "medium", + srcs = ["examples/logging_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +# Commenting out for now because it is not idempotent +# py_test( +# name = "mlflow_example", +# size = "medium", +# srcs = ["examples/mlflow_example.py"], +# deps = [":tune_lib"], +# tags = ["exclusive", "example"] +# ) + +py_test( + name = "mnist_pytorch", + size = "small", + srcs = ["examples/mnist_pytorch.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example", "pytorch"], + args = ["--smoke-test"] +) + +py_test( + name = "mnist_pytorch_lightning", + size = "medium", + srcs = ["examples/mnist_pytorch_lightning.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example", "pytorch"], + args = ["--smoke-test"] +) + +py_test( + name = "mnist_pytorch_trainable", + size = "small", + srcs = ["examples/mnist_pytorch_trainable.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example", "pytorch"], + args = ["--smoke-test"] +) + +py_test( + name = "nevergrad_example", + size = "medium", + srcs = ["examples/nevergrad_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "pbt_convnet_example", + size = "medium", + srcs = ["examples/pbt_convnet_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "pbt_example", + size = "medium", + srcs = ["examples/pbt_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "pbt_function", + size = "medium", + srcs = ["examples/pbt_function.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "pbt_memnn_example", + size = "medium", + srcs = ["examples/pbt_memnn_example.py"], + deps = [":tune_lib"], + tags = ["flaky", "exclusive", "example"], + args = ["--smoke-test"] +) + +# Requires GPUs. Add smoke test? +# py_test( +# name = "pbt_ppo_example", +# size = "medium", +# srcs = ["examples/pbt_ppo_example.py"], +# deps = [":tune_lib"], +# tags = ["exclusive", "example"], +# args = ["--smoke-test"] +# ) + +# Requires GPUs. Add smoke test? +# py_test( +# name = "pbt_tune_cifar10_with_keras", +# size = "medium", +# srcs = ["examples/pbt_tune_cifar10_with_keras.py"], +# deps = [":tune_lib"], +# tags = ["exclusive", "example"], +# args = ["--smoke-test"] +# ) + +# Needs SigOpt API key. +# py_test( +# name = "sigopt_example", +# size = "medium", +# srcs = ["examples/sigopt_example.py"], +# deps = [":tune_lib"], +# tags = ["exclusive", "example"], +# args = ["--smoke-test"] +# ) + +py_test( + name = "skopt_example", + size = "medium", + srcs = ["examples/skopt_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "tf_mnist_example", + size = "medium", + srcs = ["examples/tf_mnist_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example", "tf"], + args = ["--smoke-test"] +) + +# Downloads too much data. +# py_test( +# name = "tune_cifar10_gluon", +# size = "medium", +# srcs = ["examples/tune_cifar10_gluon.py"], +# deps = [":tune_lib"], +# tags = ["exclusive", "example"], +# args = ["--model SqueezeNet1.0", "--smoke-test"] +# ) + +py_test( + name = "tune_mnist_keras", + size = "medium", + srcs = ["examples/tune_mnist_keras.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + +py_test( + name = "xgboost_example", + size = "small", + srcs = ["examples/xgboost_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"] +) + +py_test( + name = "zoopt_example", + size = "small", + srcs = ["examples/zoopt_example.py"], + deps = [":tune_lib"], + tags = ["exclusive", "example"], + args = ["--smoke-test"] +) + # This is a dummy test dependency that causes the above tests to be # re-run if any of these files changes. py_library( diff --git a/python/ray/tune/examples/pbt_memnn_example.py b/python/ray/tune/examples/pbt_memnn_example.py index bd2e246e2..65ac2cd78 100644 --- a/python/ray/tune/examples/pbt_memnn_example.py +++ b/python/ray/tune/examples/pbt_memnn_example.py @@ -22,7 +22,7 @@ import tarfile import numpy as np import re -from ray.tune import Trainable +from ray import tune def tokenize(sent): @@ -94,7 +94,7 @@ def vectorize_stories(word_idx, story_maxlen, query_maxlen, data): pad_sequences(queries, maxlen=query_maxlen), np.array(answers)) -def read_data(): +def read_data(finish_fast=False): # Get the file try: path = get_file( @@ -125,11 +125,13 @@ def read_data(): with tarfile.open(path) as tar: train_stories = get_stories(tar.extractfile(challenge.format("train"))) test_stories = get_stories(tar.extractfile(challenge.format("test"))) - + if finish_fast: + train_stories = train_stories[:64] + test_stories = test_stories[:64] return train_stories, test_stories -class MemNNModel(Trainable): +class MemNNModel(tune.Trainable): def build_model(self): """Helper method for creating the model""" vocab = set() @@ -216,7 +218,8 @@ class MemNNModel(Trainable): def setup(self, config): with FileLock(os.path.expanduser("~/.tune.lock")): - self.train_stories, self.test_stories = read_data() + self.train_stories, self.test_stories = read_data( + config["finish_fast"]) model = self.build_model() rmsprop = RMSprop( lr=self.config.get("lr", 1e-3), rho=self.config.get("rho", 0.9)) @@ -255,33 +258,34 @@ class MemNNModel(Trainable): if __name__ == "__main__": import ray - from ray.tune import Trainable, run from ray.tune.schedulers import PopulationBasedTraining parser = argparse.ArgumentParser() parser.add_argument( "--smoke-test", action="store_true", help="Finish quickly for testing") args, _ = parser.parse_known_args() - ray.init() + ray.init(num_cpus=2) + read_data() pbt = PopulationBasedTraining( time_attr="training_iteration", metric="mean_accuracy", mode="max", - perturbation_interval=5, + perturbation_interval=2, hyperparam_mutations={ "dropout": lambda: np.random.uniform(0, 1), "lr": lambda: 10**np.random.randint(-10, 0), "rho": lambda: np.random.uniform(0, 1) }) - results = run( + results = tune.run( MemNNModel, name="pbt_babi_memnn", scheduler=pbt, - stop={"training_iteration": 10 if args.smoke_test else 100}, - num_samples=4, + stop={"training_iteration": 4 if args.smoke_test else 100}, + num_samples=2, config={ + "finish_fast": args.smoke_test, "batch_size": 32, "epochs": 1, "dropout": 0.3, diff --git a/python/ray/tune/examples/pbt_ppo_example.py b/python/ray/tune/examples/pbt_ppo_example.py index 1478541f6..d084f6214 100755 --- a/python/ray/tune/examples/pbt_ppo_example.py +++ b/python/ray/tune/examples/pbt_ppo_example.py @@ -6,7 +6,7 @@ to run concurrently, otherwise PBT will round-robin train the trials which is less efficient (or you can set {"gpu": 0} to use CPUs for SGD instead). Note that Tune in general does not need 8 GPUs, and this is just a more -computationally demainding example. +computationally demanding example. """ import random diff --git a/python/ray/util/sgd/BUILD b/python/ray/util/sgd/BUILD new file mode 100644 index 000000000..284897c22 --- /dev/null +++ b/python/ray/util/sgd/BUILD @@ -0,0 +1,213 @@ +# -------------------------------------------------------------------- +# Tests from the python/ray/util/sgd/tests directory. +# Please keep these sorted alphabetically. +# -------------------------------------------------------------------- +py_test( + name = "test_tensorflow", + size = "small", + srcs = ["tests/test_tensorflow.py"], + tags = ["exclusive", "tf"], + deps = [":sgd_lib"], +) + +py_test( + name = "test_torch", + size = "large", + srcs = ["tests/test_torch.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], +) + +py_test( + name = "test_torch_runner", + size = "small", + srcs = ["tests/test_torch_runner.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], +) + +# -------------------------------------------------------------------- +# Tests from the python/ray/util/sgd/tf/examples directory. +# Please keep these sorted alphabetically. +# -------------------------------------------------------------------- +py_test( + name = "cifar_tf_example_1", + size = "medium", + main = "tf/examples/cifar_tf_example.py", + srcs = ["tf/examples/cifar_tf_example.py"], + tags = ["exclusive", "tf"], + deps = [":sgd_lib"], + args = ["--smoke-test", "--num-replicas=1"] +) + +py_test( + name = "cifar_tf_example_2", + size = "medium", + main = "tf/examples/cifar_tf_example.py", + srcs = ["tf/examples/cifar_tf_example.py"], + tags = ["exclusive", "tf"], + deps = [":sgd_lib"], + args = ["--smoke-test", "--num-replicas=2"] +) + +py_test( + name = "cifar_tf_example_2b", + size = "small", + main = "tf/examples/cifar_tf_example.py", + srcs = ["tf/examples/cifar_tf_example.py"], + tags = ["exclusive", "tf"], + deps = [":sgd_lib"], + args = ["--smoke-test", "--num-replicas=2", "--augment-data"] +) + +py_test( + name = "tensorflow_train_example_1", + size = "small", + main = "tf/examples/tensorflow_train_example.py", + srcs = ["tf/examples/tensorflow_train_example.py"], + tags = ["exclusive", "tf"], + deps = [":sgd_lib"], + args = ["--num-replicas=1"] +) + +py_test( + name = "tensorflow_train_example_2", + size = "small", + main = "tf/examples/tensorflow_train_example.py", + srcs = ["tf/examples/tensorflow_train_example.py"], + tags = ["exclusive", "tf"], + deps = [":sgd_lib"], + args = ["--num-replicas=2"] +) + +py_test( + name = "tensorflow_train_example_tune", + size = "small", + main = "tf/examples/tensorflow_train_example.py", + srcs = ["tf/examples/tensorflow_train_example.py"], + tags = ["exclusive", "tf"], + deps = [":sgd_lib"], + args = ["--tune"] +) + +# -------------------------------------------------------------------- +# Tests from the python/ray/util/sgd/torch/examples directory. +# Does not include subdirectories. +# Please keep these sorted alphabetically. +# -------------------------------------------------------------------- +py_test( + name = "cifar_pytorch_example_1", + size = "medium", + main = "torch/examples/cifar_pytorch_example.py", + srcs = ["torch/examples/cifar_pytorch_example.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--smoke-test", "--num-workers=1"] +) + +py_test( + name = "cifar_pytorch_example_2", + size = "medium", + main = "torch/examples/cifar_pytorch_example.py", + srcs = ["torch/examples/cifar_pytorch_example.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--smoke-test", "--num-workers=2"] +) + +py_test( + name = "cifar_pytorch_pbt", + size = "medium", + srcs = ["torch/examples/cifar_pytorch_pbt.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--smoke-test"] +) + +py_test( + name = "dcgan", + size = "small", + srcs = ["torch/examples/dcgan.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--smoke-test", "--num-workers=2"] +) + +py_test( + name = "raysgd_torch_signatures", + size = "small", + srcs = ["torch/examples/raysgd_torch_signatures.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"] +) + +py_test( + name = "train_example_1", + size = "small", + main = "torch/examples/train_example.py", + srcs = ["torch/examples/train_example.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--num-workers=1"] +) + +py_test( + name = "train_example_2", + size = "small", + main = "torch/examples/train_example.py", + srcs = ["torch/examples/train_example.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--num-workers=2"] +) + +py_test( + name = "tune_example_1", + size = "small", + main = "torch/examples/tune_example.py", + srcs = ["torch/examples/tune_example.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--num-workers=1"] +) + +py_test( + name = "tune_example_2", + size = "small", + main = "torch/examples/tune_example.py", + srcs = ["torch/examples/tune_example.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--num-workers=2"] +) + +# -------------------------------------------------------------------- +# Tests from the python/ray/util/sgd/torch/examples/* directories. +# Only covers subdirectories. +# Please keep these sorted alphabetically. +# -------------------------------------------------------------------- +py_test( + name = "benchmark", + size = "small", + srcs = ["torch/examples/benchmarks/benchmark.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--smoke-test"] +) + +py_test( + name = "image_models", + size = "small", + main = "torch/examples/image_models/train.py", + srcs = ["torch/examples/image_models/train.py"], + tags = ["exclusive", "pytorch"], + deps = [":sgd_lib"], + args = ["--no-gpu", "--mock-data", "--smoke-test", "--ray-num-workers=2", "--model=mobilenetv3_small_075", "data"] +) + +# This is a dummy test dependency that causes the above tests to be +# re-run if any of these files changes. +py_library( + name = "sgd_lib", + srcs = glob(["**/*.py"], exclude=["tests/*.py"]), +) \ No newline at end of file