Fix frequent failure of Jenkins CI. (#2490)

test/jenkins_tests/multi_node_docker_test.py

@@ -3,8 +3,9 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import numpy as np
+import datetime
 import os
+import random
 import re
 import signal
 import subprocess
@@ -32,7 +33,15 @@ def wait_for_output(proc):
     Returns:
         A tuple of the stdout and stderr of the process as strings.
     """
-    stdout_data, stderr_data = proc.communicate()
+    try:
+        # NOTE: This test must be run with Python 3.
+        stdout_data, stderr_data = proc.communicate(timeout=200)
+    except subprocess.TimeoutExpired:
+        # Timeout: kill the process.
+        # Get the remaining message from PIPE for debugging purpose.
+        print("Killing process because it timed out.")
+        proc.kill()
+        stdout_data, stderr_data = proc.communicate()
 
     if stdout_data is not None:
         try:
@@ -71,11 +80,12 @@ class DockerRunner(object):
             head node.
     """
 
-    def __init__(self):
+    def __init__(self, use_raylet):
         """Initialize the DockerRunner."""
         self.head_container_id = None
         self.worker_container_ids = []
         self.head_container_ip = None
+        self.use_raylet = use_raylet
 
     def _get_container_id(self, stdout_data):
         """Parse the docker container ID from stdout_data.
@@ -139,6 +149,8 @@ class DockerRunner(object):
             "--num-cpus={}".format(num_cpus), "--num-gpus={}".format(num_gpus),
             "--no-ui"
         ])
+        if self.use_raylet:
+            command.append("--use-raylet")
         print("Starting head node with command:{}".format(command))
 
         proc = subprocess.Popen(
@@ -165,6 +177,8 @@ class DockerRunner(object):
             "--redis-address={:s}:6379".format(self.head_container_ip),
             "--num-cpus={}".format(num_cpus), "--num-gpus={}".format(num_gpus)
         ])
+        if self.use_raylet:
+            command.append("--use-raylet")
         print("Starting worker node with command:{}".format(command))
         proc = subprocess.Popen(
             command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -289,13 +303,16 @@ class DockerRunner(object):
         Raises:
             Exception: An exception is raised if the timeout expires.
         """
+        print("Multi-node docker test started at: {}".format(
+            datetime.datetime.now()))
         all_container_ids = (
             [self.head_container_id] + self.worker_container_ids)
         if driver_locations is None:
             driver_locations = [
-                np.random.randint(0, len(all_container_ids))
-                for _ in range(num_drivers)
+                random.randrange(0, len(all_container_ids))
+                for i in range(num_drivers)
             ]
+        print("driver_locations: {}".format(driver_locations))
 
         # Define a signal handler and set an alarm to go off in
         # timeout_seconds.
@@ -308,13 +325,19 @@ class DockerRunner(object):
 
         # Start the different drivers.
         driver_processes = []
+        if self.use_raylet:
+            use_raylet_env = 1
+        else:
+            use_raylet_env = 0
         for i in range(len(driver_locations)):
             # Get the container ID to run the ith driver in.
             container_id = all_container_ids[driver_locations[i]]
             command = [
-                "docker", "exec", container_id, "/bin/bash", "-c",
-                ("RAY_REDIS_ADDRESS={}:6379 RAY_DRIVER_INDEX={} python "
-                 "{}".format(self.head_container_ip, i, test_script))
+                "docker", "exec", container_id, "/bin/bash",
+                "-c", ("RAY_REDIS_ADDRESS={}:6379 RAY_DRIVER_INDEX={} "
+                       "RAY_USE_XRAY={} python {}".format(
+                           self.head_container_ip, i, use_raylet_env,
+                           test_script))
             ]
             print("Starting driver with command {}.".format(test_script))
             # Start the driver.
@@ -322,7 +345,6 @@ class DockerRunner(object):
                 command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             driver_processes.append(p)
 
-        # Wait for the drivers to finish.
         results = []
         for p in driver_processes:
             stdout_data, stderr_data = wait_for_output(p)
@@ -337,7 +359,8 @@ class DockerRunner(object):
 
         # Disable the alarm.
         signal.alarm(0)
-
+        print("Multi-node docker test ended at: {}".format(
+            datetime.datetime.now()))
         return results
 
 
@@ -381,6 +404,8 @@ if __name__ == "__main__":
         "--development-mode",
         action="store_true",
         help="use local copies of the test scripts")
+    parser.add_argument(
+        "--use-raylet", action="store_true", help="use raylet mode in Docker")
     args = parser.parse_args()
 
     # Parse the number of CPUs and GPUs to use for each worker.
@@ -394,7 +419,7 @@ if __name__ == "__main__":
     driver_locations = (None if args.driver_locations is None else
                         [int(i) for i in args.driver_locations.split(",")])
 
-    d = DockerRunner()
+    d = DockerRunner(args.use_raylet)
     d.start_ray(
         docker_image=args.docker_image,
         mem_size=args.mem_size,