Test dying_worker_get and dying_worker_wait for xray. (#2997)

This tests the case in which a worker is blocked in a call to ray.get or ray.wait, and then the worker dies. Then later, the object that the worker was waiting for becomes available. We need to make sure not to try to send a message to the dead worker and then die. Related to #2790.

python/ray/test/test_utils.py

@@ -6,6 +6,7 @@ import json
 import os
 import redis
 import subprocess
+import sys
 import tempfile
 import time
 
@@ -147,3 +148,40 @@ def run_and_get_output(command):
         with open(tmp.name, 'r') as f:
             result = f.readlines()
             return "\n".join(result)
+
+
+def run_string_as_driver(driver_script):
+    """Run a driver as a separate process.
+
+    Args:
+        driver_script: A string to run as a Python script.
+
+    Returns:
+        The script's output.
+    """
+    # Save the driver script as a file so we can call it using subprocess.
+    with tempfile.NamedTemporaryFile() as f:
+        f.write(driver_script.encode("ascii"))
+        f.flush()
+        out = ray.utils.decode(
+            subprocess.check_output([sys.executable, f.name]))
+    return out
+
+
+def run_string_as_driver_nonblocking(driver_script):
+    """Start a driver as a separate process and return immediately.
+
+    Args:
+        driver_script: A string to run as a Python script.
+
+    Returns:
+        A handle to the driver process.
+    """
+    # Save the driver script as a file so we can call it using subprocess. We
+    # do not delete this file because if we do then it may get removed before
+    # the Python process tries to run it.
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(driver_script.encode("ascii"))
+        f.flush()
+        return subprocess.Popen(
+            [sys.executable, f.name], stdout=subprocess.PIPE)