[tune] Fix Jenkins tests (#6028)

ci/jenkins_tests/run_tune_tests.sh

@@ -55,7 +55,7 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE}
 $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     bash -c 'pip install -U tensorflow==1.12 && python /ray/python/ray/tune/tests/test_logger.py'
 
-$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} -e MPLBACKEND=Agg $DOCKER_SHA \
     python /ray/python/ray/tune/tests/tutorial.py
 
 $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \

docker/examples/Dockerfile

@@ -4,7 +4,8 @@ FROM ray-project/deploy
 
 # This updates numpy to 1.14 and mutes errors from other libraries
 RUN conda install -y numpy
-RUN apt-get install -y zlib1g-dev
+# Needed to run Tune example with a 'plot' call - which does not actually render a plot, but throws an error.
+RUN apt-get install -y zlib1g-dev libgl1-mesa-dev
 # The following is needed to support TensorFlow 1.14
 RUN conda remove -y --force wrapt
 RUN pip install -U pip

docker/tune_test/Dockerfile

@@ -9,7 +9,8 @@ RUN pip install -U pip
 RUN pip install -U https://ray-wheels.s3-us-west-2.amazonaws.com/latest/ray-0.8.0.dev6-cp36-cp36m-manylinux1_x86_64.whl || pip install -U https://ray-wheels.s3-us-west-2.amazonaws.com/latest/ray-0.8.0.dev5-cp36-cp36m-manylinux1_x86_64.whl
 RUN pip install -U boto3
 # We install this after the latest wheels -- this should not override the latest wheels.
-RUN apt-get install -y zlib1g-dev
+# Needed to run Tune example with a 'plot' call - which does not actually render a plot, but throws an error.
+RUN apt-get install -y zlib1g-dev libgl1-mesa-dev
 # The following is needed to support TensorFlow 1.14
 RUN conda remove -y --force wrapt
 RUN pip install gym[atari]==0.10.11 opencv-python-headless tensorflow lz4 keras pytest-timeout smart_open torch torchvision

python/ray/tune/tests/test_cluster.py

@@ -370,8 +370,8 @@ def test_cluster_down_simple(start_connected_cluster, tmpdir):
     assert all(t.status == Trial.RUNNING for t in runner.get_trials())
     runner.checkpoint()
 
-    cluster.shutdown()
     ray.shutdown()
+    cluster.shutdown()
 
     cluster = _start_new_cluster()
     runner = TrialRunner(resume="LOCAL", local_checkpoint_dir=dirpath)
@@ -385,6 +385,7 @@ def test_cluster_down_simple(start_connected_cluster, tmpdir):
         runner.step()
 
     assert all(t.status == Trial.TERMINATED for t in runner.get_trials())
+    ray.shutdown()
     cluster.shutdown()
 
 
@@ -425,6 +426,7 @@ def test_cluster_down_full(start_connected_cluster, tmpdir):
         all_experiments, resume=True, raise_on_failed_trial=False)
     assert len(trials) == 4
     assert all(t.status in [Trial.TERMINATED, Trial.ERROR] for t in trials)
+    ray.shutdown()
     cluster.shutdown()
 
 
@@ -487,6 +489,7 @@ tune.run(
         },
         resume=True)
     assert all(t.status == Trial.TERMINATED for t in trials2)
+    ray.shutdown()
     cluster.shutdown()
 
 
@@ -588,4 +591,5 @@ tune.run(
         raise_on_failed_trial=False)
     assert all(t.status == Trial.TERMINATED for t in trials2)
     assert {t.trial_id for t in trials2} == {t.trial_id for t in trials}
+    ray.shutdown()
     cluster.shutdown()