From b052bcf1fc8e35e3af16b634dff2fd83bb75ca4c Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Fri, 22 Nov 2019 13:58:50 -0800 Subject: [PATCH] Bazelify tune tests in travis (#6219) --- .travis.yml | 14 +- python/ray/tune/BUILD | 123 ++++++++++++++++++ python/ray/tune/result.py | 3 +- python/ray/tune/tests/test_automl_searcher.py | 4 + .../ray/tune/tests/test_checkpoint_manager.py | 6 + python/ray/tune/tests/test_cluster.py | 6 + python/ray/tune/tests/test_commands.py | 7 + python/ray/tune/tests/test_experiment.py | 4 + python/ray/tune/tests/test_track.py | 4 + 9 files changed, 162 insertions(+), 9 deletions(-) create mode 100644 python/ray/tune/BUILD diff --git a/.travis.yml b/.travis.yml index 51766c1c0..3295df708 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,18 +5,18 @@ dist: xenial matrix: include: - os: linux - env: PYTHON=2.7 PYTHONWARNINGS=ignore + env: BAZEL_PYTHON_VERSION=PY2 PYTHON=2.7 PYTHONWARNINGS=ignore - os: linux - env: PYTHON=3.5 PYTHONWARNINGS=ignore + env: BAZEL_PYTHON_VERSION=PY3 PYTHON=3.5 PYTHONWARNINGS=ignore - os: osx osx_image: xcode7 - env: PYTHON=2.7 PYTHONWARNINGS=ignore + env: BAZEL_PYTHON_VERSION=PY2 PYTHON=2.7 PYTHONWARNINGS=ignore - os: osx osx_image: xcode7 - env: PYTHON=3.5 PYTHONWARNINGS=ignore + env: BAZEL_PYTHON_VERSION=PY3 PYTHON=3.5 PYTHONWARNINGS=ignore - os: linux env: @@ -173,10 +173,8 @@ script: # module is only found if the test directory is in the PYTHONPATH. # - export PYTHONPATH="$PYTHONPATH:./ci/" - # ray tune tests - - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/tune/tests/test_dependency.py; fi - # `cluster_tests.py` runs on Jenkins, not Travis. - - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then python -m pytest -v --durations=10 --timeout=300 --ignore=python/ray/tune/tests/test_cluster.py --ignore=python/ray/tune/tests/test_logger.py --ignore=python/ray/tune/tests/test_tune_restore.py --ignore=python/ray/tune/tests/test_actor_reuse.py python/ray/tune/tests; fi + # ray tune tests, except tests tagged jenkins only. + - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then bazel test --spawn_strategy=local --python_version=$BAZEL_PYTHON_VERSION --incompatible_allow_python_version_transitions=false --incompatible_py3_is_default=false --show_progress_rate_limit=120 --test_output=errors --test_tag_filters=-jenkins_only python/ray/tune/...; fi # ray serve tests - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then python -c 'import sys;exit(sys.version_info>=(3,5))' || python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/serve/tests; fi diff --git a/python/ray/tune/BUILD b/python/ray/tune/BUILD new file mode 100644 index 000000000..3b1c9840f --- /dev/null +++ b/python/ray/tune/BUILD @@ -0,0 +1,123 @@ +py_test( + name = "test_actor_reuse", + size = "medium", + srcs = ["tests/test_actor_reuse.py"], + tags = ["jenkins_only"], +) + +py_test( + name = "test_automl_searcher", + size = "small", + srcs = ["tests/test_automl_searcher.py"], +) + +py_test( + name = "test_checkpoint_manager", + size = "small", + srcs = ["tests/test_checkpoint_manager.py"], + deps = [":tune_lib"], +) + +py_test( + name = "test_cluster", + size = "large", + srcs = ["tests/test_cluster.py"], + deps = [":tune_lib"], + tags = ["jenkins_only", "exclusive"], +) + +py_test( + name = "test_commands", + size = "medium", + srcs = ["tests/test_commands.py"], + deps = [":tune_lib"], + tags = ["exclusive"], +) + +py_test( + name = "test_dependency", + size = "small", + srcs = ["tests/test_dependency.py"], + deps = [":tune_lib"], +) + +py_test( + name = "test_experiment_analysis", + size = "medium", + srcs = ["tests/test_experiment_analysis.py"], + deps = [":tune_lib"], +) + +py_test( + name = "test_experiment", + size = "small", + srcs = ["tests/test_experiment.py"], + deps = [":tune_lib"], +) + +py_test( + name = "test_logger", + size = "small", + srcs = ["tests/test_logger.py"], + deps = [":tune_lib"], + tags = ["jenkins_only"], +) + +py_test( + name = "test_ray_trial_executor", + size = "medium", + srcs = ["tests/test_ray_trial_executor.py"], + deps = [":tune_lib"], +) + +py_test( + name = "test_track", + size = "small", + srcs = ["tests/test_track.py"], + deps = [":tune_lib"], +) + +py_test( + name = "test_trial_runner", + size = "large", + srcs = ["tests/test_trial_runner.py"], + deps = [":tune_lib"], + tags = ["exclusive"], +) + +py_test( + name = "test_trial_scheduler", + size = "medium", + srcs = ["tests/test_trial_scheduler.py"], + deps = [":tune_lib"], +) + +py_test( + name = "test_tune_restore", + size = "large", + srcs = ["tests/test_tune_restore.py"], + deps = [":tune_lib"], + tags = ["jenkins_only", "exclusive"], +) + +py_test( + name = "test_tune_save_restore", + size = "large", + srcs = ["tests/test_tune_save_restore.py"], + deps = [":tune_lib"], + tags = ["exclusive"], +) + +py_test( + name = "test_tune_server", + size = "medium", + srcs = ["tests/test_tune_server.py"], + deps = [":tune_lib"], +) + +# This is a dummy test dependency that causes the above tests to be +# re-run if any of these files changes. +py_library( + name="tune_lib", + srcs = glob(["**/*.py"], exclude=["tests/*.py"]), +) diff --git a/python/ray/tune/result.py b/python/ray/tune/result.py index fb4c4fcdc..aec191535 100644 --- a/python/ray/tune/result.py +++ b/python/ray/tune/result.py @@ -70,7 +70,8 @@ DEFAULT_RESULT_KEYS = (TRAINING_ITERATION, TIME_TOTAL_S, TIMESTEPS_TOTAL, RESULT_DUPLICATE = "__duplicate__" # Where Tune writes result files by default -DEFAULT_RESULTS_DIR = (os.environ.get("TUNE_RESULT_DIR") +DEFAULT_RESULTS_DIR = (os.environ.get("TEST_TMPDIR") + or os.environ.get("TUNE_RESULT_DIR") or os.path.expanduser("~/ray_results")) # Meta file about status under each experiment directory, can be diff --git a/python/ray/tune/tests/test_automl_searcher.py b/python/ray/tune/tests/test_automl_searcher.py index 1f2818e13..49f03af87 100644 --- a/python/ray/tune/tests/test_automl_searcher.py +++ b/python/ray/tune/tests/test_automl_searcher.py @@ -67,3 +67,7 @@ class AutoMLSearcherTest(unittest.TestCase): best_trial = searcher.get_best_trial() self.assertEqual(best_trial, trials[-1]) self.assertEqual(best_trial.best_result["reward"], 3 + 10 - 1) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/ray/tune/tests/test_checkpoint_manager.py b/python/ray/tune/tests/test_checkpoint_manager.py index fd1975476..ec6741e12 100644 --- a/python/ray/tune/tests/test_checkpoint_manager.py +++ b/python/ray/tune/tests/test_checkpoint_manager.py @@ -104,3 +104,9 @@ class CheckpointManagerTest(unittest.TestCase): log_error_mock.assert_called_once() # The newest checkpoint should still be set despite this error. assert checkpoint_manager.newest_checkpoint == no_attr_checkpoint + + +if __name__ == "__main__": + import pytest + import sys + sys.exit(pytest.main(["-v", "-s", __file__])) diff --git a/python/ray/tune/tests/test_cluster.py b/python/ray/tune/tests/test_cluster.py index bc88f5743..cf1024db9 100644 --- a/python/ray/tune/tests/test_cluster.py +++ b/python/ray/tune/tests/test_cluster.py @@ -593,3 +593,9 @@ tune.run( assert {t.trial_id for t in trials2} == {t.trial_id for t in trials} ray.shutdown() cluster.shutdown() + + +if __name__ == "__main__": + import pytest + import sys + sys.exit(pytest.main(["-v", "-s", __file__])) diff --git a/python/ray/tune/tests/test_commands.py b/python/ray/tune/tests/test_commands.py index 8234ffd79..e82f21d34 100644 --- a/python/ray/tune/tests/test_commands.py +++ b/python/ray/tune/tests/test_commands.py @@ -156,3 +156,10 @@ def test_lsx(start_ray, tmpdir): lines = output.captured assert sum("1" in line for line in lines) >= num_experiments assert len(lines) == 3 + num_experiments + 1 + + +if __name__ == "__main__": + # Make click happy in bazel. + os.environ["LC_ALL"] = "en_US.UTF-8" + os.environ["LANG"] = "en_US.UTF-8" + sys.exit(pytest.main([__file__])) diff --git a/python/ray/tune/tests/test_experiment.py b/python/ray/tune/tests/test_experiment.py index 8298a2d27..841295680 100644 --- a/python/ray/tune/tests/test_experiment.py +++ b/python/ray/tune/tests/test_experiment.py @@ -59,3 +59,7 @@ class ExperimentTest(unittest.TestCase): def testConvertExperimentIncorrect(self): self.assertRaises(TuneError, lambda: convert_to_experiment_list("hi")) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/ray/tune/tests/test_track.py b/python/ray/tune/tests/test_track.py index 5a1687100..7bfc8683e 100644 --- a/python/ray/tune/tests/test_track.py +++ b/python/ray/tune/tests/test_track.py @@ -82,3 +82,7 @@ class TrackApiTest(unittest.TestCase): self.assertTrue(_check_json_val(result_path, "test", 1)) track.log(iteration=1, test=2) self.assertTrue(_check_json_val(result_path, "test", 2)) + + +if __name__ == "__main__": + unittest.main(verbosity=2)