[tune] move jenkins tests to travis (#9609)

Co-authored-by: Richard Liaw <rliaw@berkeley.edu>
Co-authored-by: Kai Fricke <kai@anyscale.com>

.travis.yml

@@ -247,6 +247,82 @@ matrix:
       script:
         - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/...
 
+
+    # Tune: Tests and examples.
+    - os: linux
+      env:
+        - TUNE_TESTING=1
+        - PYTHON=3.6
+        - TF_VERSION=2.1.0
+        - TFP_VERSION=0.8
+        - TORCH_VERSION=1.5
+        - PYTHONWARNINGS=ignore
+      install:
+        - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED
+      before_script:
+        - . ./ci/travis/ci.sh build
+      script:
+        - ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only,-example python/ray/tune/...
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=example,-tf,-pytorch,-py37,-flaky python/ray/tune/...
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-flaky python/ray/tune/...
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-flaky python/ray/tune/...
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-py37,flaky python/ray/tune/...
+
+    # SGD: Tests and examples.
+    - os: linux
+      env:
+        - SGD_TESTING=1
+        - PYTHON=3.6
+        - TF_VERSION=2.1.0
+        - TFP_VERSION=0.8
+        - TORCH_VERSION=1.5
+        - PYTHONWARNINGS=ignore
+      install:
+        - . ./ci/travis/ci.sh init RAY_CI_SGD_AFFECTED
+      before_script:
+        - . ./ci/travis/ci.sh build
+      script:
+        # No such tests at the moment:
+        # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,-pytorch,-py37 python/ray/util/sgd/...
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tf,-pytorch,-py37 python/ray/util/sgd/...
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,pytorch,-py37 python/ray/util/sgd/...
+
+    # Docs: Tests and examples.
+    - os: linux
+      env:
+        - DOC_TESTING=1
+        - PYTHON=3.6
+        - TF_VERSION=2.1.0
+        - TFP_VERSION=0.8
+        - TORCH_VERSION=1.5
+        - PYTHONWARNINGS=ignore
+      install:
+        - . ./ci/travis/ci.sh init RAY_CI_PYTHON_AFFECTED
+      before_script:
+        - . ./ci/travis/ci.sh build
+      script:
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,-pytorch,-py37 doc/...
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tf,-pytorch,-py37 doc/...
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-tf,pytorch,-py37 doc/...
+
+    # Tune/SGD/Docs: Python >= 3.7 tests and examples.
+    - os: linux
+      env:
+        - PYTHON=3.7 TUNE_TESTING=1
+        - TF_VERSION=2.1.0
+        - TFP_VERSION=0.8
+        - TORCH_VERSION=1.5
+        - PYTHONWARNINGS=ignore
+      install:
+        - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED,RAY_CI_SGD_AFFECTED
+      before_script:
+        - . ./ci/travis/ci.sh build
+      script:
+        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 python/ray/tune/...
+        # There are no python 3.7 tests for RaySGD/Docs at the moment
+        # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 python/ray/util/sgd/...
+        # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 doc/...
+
     # Cpp worker test
     - os: linux
       env:
@@ -281,7 +357,6 @@ script:
 
   # bazel python tests. This should be run last to keep its logs at the end of travis logs.
   - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/...; fi
-  - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tune/...; fi
   # NO MORE TESTS BELOW, keep them above.
 
 deploy:

ci/travis/determine_tests_to_run.py

@@ -33,6 +33,7 @@ def list_changed_files(commit_range):
 if __name__ == "__main__":
 
     RAY_CI_TUNE_AFFECTED = 0
+    RAY_CI_SGD_AFFECTED = 0
     RAY_CI_ONLY_RLLIB_AFFECTED = 0  # Whether only RLlib is affected.
     RAY_CI_RLLIB_AFFECTED = 0  # Whether RLlib minimal tests should be run.
     RAY_CI_RLLIB_FULL_AFFECTED = 0  # Whether full RLlib tests should be run.
@@ -73,6 +74,10 @@ if __name__ == "__main__":
                 RAY_CI_RLLIB_FULL_AFFECTED = 1
                 RAY_CI_LINUX_WHEELS_AFFECTED = 1
                 RAY_CI_MACOS_WHEELS_AFFECTED = 1
+            elif changed_file.startswith("python/ray/util/sgd"):
+                RAY_CI_SGD_AFFECTED = 1
+                RAY_CI_LINUX_WHEELS_AFFECTED = 1
+                RAY_CI_MACOS_WHEELS_AFFECTED = 1
             elif re.match("^(python/ray/)?rllib/", changed_file):
                 RAY_CI_RLLIB_AFFECTED = 1
                 RAY_CI_RLLIB_FULL_AFFECTED = 1
@@ -86,6 +91,7 @@ if __name__ == "__main__":
                 RAY_CI_DASHBOARD_AFFECTED = 1
             elif changed_file.startswith("python/"):
                 RAY_CI_TUNE_AFFECTED = 1
+                RAY_CI_SGD_AFFECTED = 1
                 RAY_CI_RLLIB_AFFECTED = 1
                 RAY_CI_SERVE_AFFECTED = 1
                 RAY_CI_PYTHON_AFFECTED = 1
@@ -103,6 +109,7 @@ if __name__ == "__main__":
                 pass
             elif changed_file.startswith("src/"):
                 RAY_CI_TUNE_AFFECTED = 1
+                RAY_CI_SGD_AFFECTED = 1
                 RAY_CI_RLLIB_AFFECTED = 1
                 RAY_CI_SERVE_AFFECTED = 1
                 RAY_CI_JAVA_AFFECTED = 1
@@ -123,6 +130,7 @@ if __name__ == "__main__":
                 RAY_CI_STREAMING_JAVA_AFFECTED = 1
             else:
                 RAY_CI_TUNE_AFFECTED = 1
+                RAY_CI_SGD_AFFECTED = 1
                 RAY_CI_RLLIB_AFFECTED = 1
                 RAY_CI_SERVE_AFFECTED = 1
                 RAY_CI_JAVA_AFFECTED = 1
@@ -134,6 +142,7 @@ if __name__ == "__main__":
                 RAY_CI_STREAMING_JAVA_AFFECTED = 1
     else:
         RAY_CI_TUNE_AFFECTED = 1
+        RAY_CI_SGD_AFFECTED = 1
         RAY_CI_RLLIB_AFFECTED = 1
         RAY_CI_RLLIB_FULL_AFFECTED = 1
         RAY_CI_SERVE_AFFECTED = 1
@@ -149,12 +158,14 @@ if __name__ == "__main__":
             not RAY_CI_JAVA_AFFECTED and not RAY_CI_PYTHON_AFFECTED and not \
             RAY_CI_STREAMING_CPP_AFFECTED and \
             not RAY_CI_STREAMING_PYTHON_AFFECTED and \
-            not RAY_CI_STREAMING_JAVA_AFFECTED:
+            not RAY_CI_STREAMING_JAVA_AFFECTED and \
+            not RAY_CI_SGD_AFFECTED:
         RAY_CI_ONLY_RLLIB_AFFECTED = 1
 
     # Log the modified environment variables visible in console.
     print(" ".join([
         "RAY_CI_TUNE_AFFECTED={}".format(RAY_CI_TUNE_AFFECTED),
+        "RAY_CI_SGD_AFFECTED={}".format(RAY_CI_SGD_AFFECTED),
         "RAY_CI_ONLY_RLLIB_AFFECTED={}".format(RAY_CI_ONLY_RLLIB_AFFECTED),
         "RAY_CI_RLLIB_AFFECTED={}".format(RAY_CI_RLLIB_AFFECTED),
         "RAY_CI_RLLIB_FULL_AFFECTED={}".format(RAY_CI_RLLIB_FULL_AFFECTED),

ci/travis/install-dependencies.sh

@@ -279,6 +279,27 @@ install_dependencies() {
       torch=="${TORCH_VERSION-1.4}" torchvision atari_py "gym[atari]" lz4 smart_open
   fi
 
+  # Additional Tune test dependencies.
+  if [ "${TUNE_TESTING-}" = 1 ]; then
+    pip install tensorflow-probability=="${TFP_VERSION-0.8}" \
+      torch=="${TORCH_VERSION-1.4}"
+    pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt
+  fi
+
+  # Additional RaySGD test dependencies.
+  if [ "${SGD_TESTING-}" = 1 ]; then
+    pip install tensorflow-probability=="${TFP_VERSION-0.8}" \
+      torch=="${TORCH_VERSION-1.4}"
+    pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt
+  fi
+
+  # Additional Doc test dependencies.
+  if [ "${DOC_TESTING-}" = 1 ]; then
+    pip install tensorflow-probability=="${TFP_VERSION-0.8}" \
+      torch=="${TORCH_VERSION-1.4}" torchvision atari_py gym[atari] lz4 smart_open
+    pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt
+  fi
+
   # Additional streaming dependencies.
   if [ "${RAY_CI_STREAMING_PYTHON_AFFECTED}" = 1 ]; then
     pip install "msgpack>=1.0.0"

doc/BUILD

@@ -0,0 +1,56 @@
+# --------------------------------------------------------------------
+# Tests from the doc/examples directory.
+# Please keep these sorted alphabetically, but start with the
+# root directory.
+# --------------------------------------------------------------------
+py_test(
+    name = "plot_hyperparameter",
+    size = "small",
+    srcs = ["examples/plot_hyperparameter.py"],
+    tags = ["exclusive"]
+)
+
+py_test(
+    name = "plot_parameter_server",
+    size = "medium",
+    srcs = ["examples/plot_parameter_server.py"],
+    tags = ["exclusive"]
+)
+
+py_test(
+    name = "plot_pong_example",
+    size = "large",
+    srcs = ["examples/plot_pong_example.py"],
+    tags = ["exclusive"]
+)
+
+# Directory: examples/doc_code
+py_test(
+    name = "doc_code_tf_example",
+    size = "small",
+    main = "examples/doc_code/tf_example.py",
+    srcs = ["examples/doc_code/tf_example.py"],
+    tags = ["exclusive", "tf"]
+)
+
+py_test(
+    name = "doc_code_torch_example",
+    size = "small",
+    main = "examples/doc_code/torch_example.py",
+    srcs = ["examples/doc_code/torch_example.py"],
+    tags = ["exclusive", "pytorch"]
+)
+
+# --------------------------------------------------------------------
+# Tests from the doc/source/tune/_tutorials directory.
+# Please keep these sorted alphabetically.
+# --------------------------------------------------------------------
+
+py_test(
+    name = "tune_sklearn",
+    size = "medium",
+    main = "source/tune/_tutorials/tune-sklearn.py",
+    srcs = ["source/tune/_tutorials/tune-sklearn.py"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)

docker/tune_test/requirements.txt

@@ -1,8 +1,10 @@
+ax-platform
 ConfigSpace==0.4.10
 bayesian-optimization
 boto3
 dm_tree
 dragonfly-opt
+gluoncv
 gym[atari]
 h5py
 hpbandster
@@ -13,6 +15,7 @@ lightgbm
 lz4
 matplotlib
 mlflow
+mxnet
 nevergrad
 opencv-python-headless
 pandas
@@ -26,8 +29,8 @@ smart_open
 tabulate
 tensorboardX
 tensorflow_probability
-torch
-torchvision
+torch>=1.5.0
+torchvision>=0.6.0
 xgboost
 zoopt>=0.4.0
 timm

python/ray/tune/BUILD

@@ -1,3 +1,8 @@
+# --------------------------------------------------------------------
+# Tests from the python/ray/tune/tests directory.
+# Covers all tests starting with `test_`.
+# Please keep these sorted alphabetically.
+# --------------------------------------------------------------------
 py_test(
     name = "test_actor_reuse",
     size = "medium",
@@ -6,6 +11,14 @@ py_test(
     deps = [":tune_lib"],
 )
 
+py_test(
+    name = "test_api",
+    size = "large",
+    srcs = ["tests/test_api.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive"],
+)
+
 py_test(
     name = "test_automl_searcher",
     size = "small",
@@ -25,7 +38,7 @@ py_test(
     size = "large",
     srcs = ["tests/test_cluster.py"],
     deps = [":tune_lib"],
-    tags = ["jenkins_only", "exclusive"],
+    tags = ["flaky", "jenkins_only", "exclusive"],
 )
 
 py_test(
@@ -51,6 +64,13 @@ py_test(
     deps = [":tune_lib"],
 )
 
+py_test(
+    name = "test_experiment",
+    size = "small",
+    srcs = ["tests/test_experiment.py"],
+    deps = [":tune_lib"],
+)
+
 py_test(
     name = "test_experiment_analysis",
     size = "medium",
@@ -66,10 +86,11 @@ py_test(
 )
 
 py_test(
-    name = "test_experiment",
-    size = "small",
-    srcs = ["tests/test_experiment.py"],
+    name = "test_function_api",
+    size = "medium",
+    srcs = ["tests/test_function_api.py"],
     deps = [":tune_lib"],
+    tags = ["exclusive"],
 )
 
 py_test(
@@ -94,6 +115,22 @@ py_test(
     deps = [":tune_lib"],
 )
 
+py_test(
+    name = "test_run_experiment",
+    size = "medium",
+    srcs = ["tests/test_run_experiment.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive"],
+)
+
+py_test(
+    name = "test_sync",
+    size = "medium",
+    srcs = ["tests/test_sync.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive"],
+)
+
 py_test(
     name = "test_track",
     size = "small",
@@ -102,11 +139,10 @@ py_test(
 )
 
 py_test(
-    name = "test_run_experiment",
-    size = "medium",
-    srcs = ["tests/test_run_experiment.py"],
+    name = "test_trainable_util",
+    size = "small",
+    srcs = ["tests/test_trainable_util.py"],
     deps = [":tune_lib"],
-    tags = ["exclusive"],
 )
 
 py_test(
@@ -141,37 +177,6 @@ py_test(
     tags = ["exclusive"],
 )
 
-py_test(
-    name = "test_api",
-    size = "large",
-    srcs = ["tests/test_api.py"],
-    deps = [":tune_lib"],
-    tags = ["exclusive"],
-)
-
-py_test(
-    name = "test_function_api",
-    size = "medium",
-    srcs = ["tests/test_function_api.py"],
-    deps = [":tune_lib"],
-    tags = ["exclusive"],
-)
-
-py_test(
-    name = "test_sync",
-    size = "medium",
-    srcs = ["tests/test_sync.py"],
-    deps = [":tune_lib"],
-    tags = ["exclusive"],
-)
-
-py_test(
-    name = "test_trainable_util",
-    size = "small",
-    srcs = ["tests/test_trainable_util.py"],
-    deps = [":tune_lib"],
-)
-
 py_test(
     name = "test_trial_scheduler",
     size = "medium",
@@ -212,6 +217,312 @@ py_test(
     tags = ["exclusive"],
 )
 
+# --------------------------------------------------------------------
+# Tests from the python/ray/tune/tests directory.
+# Covers all remaining tests that do not start with `test_`.
+# Please keep these sorted alphabetically.
+# --------------------------------------------------------------------
+py_test(
+    name = "example",
+    size = "small",
+    srcs = ["tests/example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+)
+
+# Todo: Ensure MPLBACKEND=Agg
+py_test(
+    name = "tutorial",
+    size = "medium",
+    srcs = ["tests/tutorial.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+)
+
+# --------------------------------------------------------------------
+# Examples from the python/ray/tune/examples directory.
+# Please keep these sorted alphabetically.
+# --------------------------------------------------------------------
+py_test(
+    name = "async_hyperband_example",
+    size = "small",
+    srcs = ["examples/async_hyperband_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "ax_example",
+    size = "medium",
+    srcs = ["examples/ax_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example", "py37"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "bayesopt_example",
+    size = "medium",
+    srcs = ["examples/bayesopt_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "bohb_example",
+    size = "small",
+    srcs = ["examples/bohb_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"]
+)
+
+py_test(
+    name = "cifar10_pytorch",
+    size = "medium",
+    srcs = ["examples/cifar10_pytorch.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example", "pytorch"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "dragonfly_example",
+    size = "medium",
+    srcs = ["examples/dragonfly_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+# Does not work without awscli tool installed. Check if can be mocked.
+# py_test(
+#     name = "durable_trainable_example",
+#     size = "medium",
+#     srcs = ["examples/durable_trainable_example.py"],
+#     deps = [":tune_lib"],
+#     tags = ["exclusive", "example"],
+#     args = ["--local", "--mock-storage"]
+# )
+
+py_test(
+    name = "genetic_example",
+    size = "small",
+    srcs = ["examples/genetic_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "hyperband_example",
+    size = "medium",
+    srcs = ["examples/hyperband_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"]
+)
+
+py_test(
+    name = "hyperband_function_example",
+    size = "medium",
+    srcs = ["examples/hyperband_function_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "hyperopt_example",
+    size = "medium",
+    srcs = ["examples/hyperopt_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "lightgbm_example",
+    size = "medium",
+    srcs = ["examples/lightgbm_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"]
+)
+
+py_test(
+    name = "logging_example",
+    size = "medium",
+    srcs = ["examples/logging_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+# Commenting out for now because it is not idempotent
+# py_test(
+#     name = "mlflow_example",
+#     size = "medium",
+#     srcs = ["examples/mlflow_example.py"],
+#     deps = [":tune_lib"],
+#     tags = ["exclusive", "example"]
+# )
+
+py_test(
+    name = "mnist_pytorch",
+    size = "small",
+    srcs = ["examples/mnist_pytorch.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example", "pytorch"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "mnist_pytorch_lightning",
+    size = "medium",
+    srcs = ["examples/mnist_pytorch_lightning.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example", "pytorch"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "mnist_pytorch_trainable",
+    size = "small",
+    srcs = ["examples/mnist_pytorch_trainable.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example", "pytorch"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "nevergrad_example",
+    size = "medium",
+    srcs = ["examples/nevergrad_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "pbt_convnet_example",
+    size = "medium",
+    srcs = ["examples/pbt_convnet_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "pbt_example",
+    size = "medium",
+    srcs = ["examples/pbt_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "pbt_function",
+    size = "medium",
+    srcs = ["examples/pbt_function.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "pbt_memnn_example",
+    size = "medium",
+    srcs = ["examples/pbt_memnn_example.py"],
+    deps = [":tune_lib"],
+    tags = ["flaky", "exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+# Requires GPUs. Add smoke test?
+# py_test(
+#     name = "pbt_ppo_example",
+#     size = "medium",
+#     srcs = ["examples/pbt_ppo_example.py"],
+#     deps = [":tune_lib"],
+#     tags = ["exclusive", "example"],
+#     args = ["--smoke-test"]
+# )
+
+# Requires GPUs. Add smoke test?
+# py_test(
+#     name = "pbt_tune_cifar10_with_keras",
+#     size = "medium",
+#     srcs = ["examples/pbt_tune_cifar10_with_keras.py"],
+#     deps = [":tune_lib"],
+#     tags = ["exclusive", "example"],
+#     args = ["--smoke-test"]
+# )
+
+# Needs SigOpt API key.
+# py_test(
+#     name = "sigopt_example",
+#     size = "medium",
+#     srcs = ["examples/sigopt_example.py"],
+#     deps = [":tune_lib"],
+#     tags = ["exclusive", "example"],
+#     args = ["--smoke-test"]
+# )
+
+py_test(
+    name = "skopt_example",
+    size = "medium",
+    srcs = ["examples/skopt_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "tf_mnist_example",
+    size = "medium",
+    srcs = ["examples/tf_mnist_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example", "tf"],
+    args = ["--smoke-test"]
+)
+
+# Downloads too much data.
+# py_test(
+#     name = "tune_cifar10_gluon",
+#     size = "medium",
+#     srcs = ["examples/tune_cifar10_gluon.py"],
+#     deps = [":tune_lib"],
+#     tags = ["exclusive", "example"],
+#     args = ["--model SqueezeNet1.0", "--smoke-test"]
+# )
+
+py_test(
+    name = "tune_mnist_keras",
+    size = "medium",
+    srcs = ["examples/tune_mnist_keras.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "xgboost_example",
+    size = "small",
+    srcs = ["examples/xgboost_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"]
+)
+
+py_test(
+    name = "zoopt_example",
+    size = "small",
+    srcs = ["examples/zoopt_example.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive", "example"],
+    args = ["--smoke-test"]
+)
+
 # This is a dummy test dependency that causes the above tests to be
 # re-run if any of these files changes.
 py_library(

python/ray/tune/examples/pbt_memnn_example.py

@@ -22,7 +22,7 @@ import tarfile
 import numpy as np
 import re
 
-from ray.tune import Trainable
+from ray import tune
 
 
 def tokenize(sent):
@@ -94,7 +94,7 @@ def vectorize_stories(word_idx, story_maxlen, query_maxlen, data):
             pad_sequences(queries, maxlen=query_maxlen), np.array(answers))
 
 
-def read_data():
+def read_data(finish_fast=False):
     # Get the file
     try:
         path = get_file(
@@ -125,11 +125,13 @@ def read_data():
     with tarfile.open(path) as tar:
         train_stories = get_stories(tar.extractfile(challenge.format("train")))
         test_stories = get_stories(tar.extractfile(challenge.format("test")))
-
+    if finish_fast:
+        train_stories = train_stories[:64]
+        test_stories = test_stories[:64]
     return train_stories, test_stories
 
 
-class MemNNModel(Trainable):
+class MemNNModel(tune.Trainable):
     def build_model(self):
         """Helper method for creating the model"""
         vocab = set()
@@ -216,7 +218,8 @@ class MemNNModel(Trainable):
 
     def setup(self, config):
         with FileLock(os.path.expanduser("~/.tune.lock")):
-            self.train_stories, self.test_stories = read_data()
+            self.train_stories, self.test_stories = read_data(
+                config["finish_fast"])
         model = self.build_model()
         rmsprop = RMSprop(
             lr=self.config.get("lr", 1e-3), rho=self.config.get("rho", 0.9))
@@ -255,33 +258,34 @@ class MemNNModel(Trainable):
 
 if __name__ == "__main__":
     import ray
-    from ray.tune import Trainable, run
     from ray.tune.schedulers import PopulationBasedTraining
 
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--smoke-test", action="store_true", help="Finish quickly for testing")
     args, _ = parser.parse_known_args()
-    ray.init()
+    ray.init(num_cpus=2)
+    read_data()
 
     pbt = PopulationBasedTraining(
         time_attr="training_iteration",
         metric="mean_accuracy",
         mode="max",
-        perturbation_interval=5,
+        perturbation_interval=2,
         hyperparam_mutations={
             "dropout": lambda: np.random.uniform(0, 1),
             "lr": lambda: 10**np.random.randint(-10, 0),
             "rho": lambda: np.random.uniform(0, 1)
         })
 
-    results = run(
+    results = tune.run(
         MemNNModel,
         name="pbt_babi_memnn",
         scheduler=pbt,
-        stop={"training_iteration": 10 if args.smoke_test else 100},
-        num_samples=4,
+        stop={"training_iteration": 4 if args.smoke_test else 100},
+        num_samples=2,
         config={
+            "finish_fast": args.smoke_test,
             "batch_size": 32,
             "epochs": 1,
             "dropout": 0.3,

python/ray/tune/examples/pbt_ppo_example.py

@@ -6,7 +6,7 @@ to run concurrently, otherwise PBT will round-robin train the trials which
 is less efficient (or you can set {"gpu": 0} to use CPUs for SGD instead).
 
 Note that Tune in general does not need 8 GPUs, and this is just a more
-computationally demainding example.
+computationally demanding example.
 """
 
 import random

python/ray/util/sgd/BUILD

@@ -0,0 +1,213 @@
+# --------------------------------------------------------------------
+# Tests from the python/ray/util/sgd/tests directory.
+# Please keep these sorted alphabetically.
+# --------------------------------------------------------------------
+py_test(
+    name = "test_tensorflow",
+    size = "small",
+    srcs = ["tests/test_tensorflow.py"],
+    tags = ["exclusive", "tf"],
+    deps = [":sgd_lib"],
+)
+
+py_test(
+    name = "test_torch",
+    size = "large",
+    srcs = ["tests/test_torch.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+)
+
+py_test(
+    name = "test_torch_runner",
+    size = "small",
+    srcs = ["tests/test_torch_runner.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+)
+
+# --------------------------------------------------------------------
+# Tests from the python/ray/util/sgd/tf/examples directory.
+# Please keep these sorted alphabetically.
+# --------------------------------------------------------------------
+py_test(
+    name = "cifar_tf_example_1",
+    size = "medium",
+    main = "tf/examples/cifar_tf_example.py",
+    srcs = ["tf/examples/cifar_tf_example.py"],
+    tags = ["exclusive", "tf"],
+    deps = [":sgd_lib"],
+    args = ["--smoke-test", "--num-replicas=1"]
+)
+
+py_test(
+    name = "cifar_tf_example_2",
+    size = "medium",
+    main = "tf/examples/cifar_tf_example.py",
+    srcs = ["tf/examples/cifar_tf_example.py"],
+    tags = ["exclusive", "tf"],
+    deps = [":sgd_lib"],
+    args = ["--smoke-test", "--num-replicas=2"]
+)
+
+py_test(
+    name = "cifar_tf_example_2b",
+    size = "small",
+    main = "tf/examples/cifar_tf_example.py",
+    srcs = ["tf/examples/cifar_tf_example.py"],
+    tags = ["exclusive", "tf"],
+    deps = [":sgd_lib"],
+    args = ["--smoke-test", "--num-replicas=2", "--augment-data"]
+)
+
+py_test(
+    name = "tensorflow_train_example_1",
+    size = "small",
+    main = "tf/examples/tensorflow_train_example.py",
+    srcs = ["tf/examples/tensorflow_train_example.py"],
+    tags = ["exclusive", "tf"],
+    deps = [":sgd_lib"],
+    args = ["--num-replicas=1"]
+)
+
+py_test(
+    name = "tensorflow_train_example_2",
+    size = "small",
+    main = "tf/examples/tensorflow_train_example.py",
+    srcs = ["tf/examples/tensorflow_train_example.py"],
+    tags = ["exclusive", "tf"],
+    deps = [":sgd_lib"],
+    args = ["--num-replicas=2"]
+)
+
+py_test(
+    name = "tensorflow_train_example_tune",
+    size = "small",
+    main = "tf/examples/tensorflow_train_example.py",
+    srcs = ["tf/examples/tensorflow_train_example.py"],
+    tags = ["exclusive", "tf"],
+    deps = [":sgd_lib"],
+    args = ["--tune"]
+)
+
+# --------------------------------------------------------------------
+# Tests from the python/ray/util/sgd/torch/examples directory.
+# Does not include subdirectories.
+# Please keep these sorted alphabetically.
+# --------------------------------------------------------------------
+py_test(
+    name = "cifar_pytorch_example_1",
+    size = "medium",
+    main = "torch/examples/cifar_pytorch_example.py",
+    srcs = ["torch/examples/cifar_pytorch_example.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--smoke-test", "--num-workers=1"]
+)
+
+py_test(
+    name = "cifar_pytorch_example_2",
+    size = "medium",
+    main = "torch/examples/cifar_pytorch_example.py",
+    srcs = ["torch/examples/cifar_pytorch_example.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--smoke-test", "--num-workers=2"]
+)
+
+py_test(
+    name = "cifar_pytorch_pbt",
+    size = "medium",
+    srcs = ["torch/examples/cifar_pytorch_pbt.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "dcgan",
+    size = "small",
+    srcs = ["torch/examples/dcgan.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--smoke-test", "--num-workers=2"]
+)
+
+py_test(
+    name = "raysgd_torch_signatures",
+    size = "small",
+    srcs = ["torch/examples/raysgd_torch_signatures.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"]
+)
+
+py_test(
+    name = "train_example_1",
+    size = "small",
+    main = "torch/examples/train_example.py",
+    srcs = ["torch/examples/train_example.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--num-workers=1"]
+)
+
+py_test(
+    name = "train_example_2",
+    size = "small",
+    main = "torch/examples/train_example.py",
+    srcs = ["torch/examples/train_example.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--num-workers=2"]
+)
+
+py_test(
+    name = "tune_example_1",
+    size = "small",
+    main = "torch/examples/tune_example.py",
+    srcs = ["torch/examples/tune_example.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--num-workers=1"]
+)
+
+py_test(
+    name = "tune_example_2",
+    size = "small",
+    main = "torch/examples/tune_example.py",
+    srcs = ["torch/examples/tune_example.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--num-workers=2"]
+)
+
+# --------------------------------------------------------------------
+# Tests from the python/ray/util/sgd/torch/examples/* directories.
+# Only covers subdirectories.
+# Please keep these sorted alphabetically.
+# --------------------------------------------------------------------
+py_test(
+    name = "benchmark",
+    size = "small",
+    srcs = ["torch/examples/benchmarks/benchmark.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--smoke-test"]
+)
+
+py_test(
+    name = "image_models",
+    size = "small",
+    main = "torch/examples/image_models/train.py",
+    srcs = ["torch/examples/image_models/train.py"],
+    tags = ["exclusive", "pytorch"],
+    deps = [":sgd_lib"],
+    args = ["--no-gpu", "--mock-data", "--smoke-test", "--ray-num-workers=2", "--model=mobilenetv3_small_075", "data"]
+)
+
+# This is a dummy test dependency that causes the above tests to be
+# re-run if any of these files changes.
+py_library(
+    name = "sgd_lib",
+    srcs = glob(["**/*.py"], exclude=["tests/*.py"]),
+)