Test dying_worker_get and dying_worker_wait for xray. (#2997)

This tests the case in which a worker is blocked in a call to ray.get or ray.wait, and then the worker dies. Then later, the object that the worker was waiting for becomes available. We need to make sure not to try to send a message to the dead worker and then die. Related to #2790.

test/multi_node_test.py

@@ -5,49 +5,11 @@ from __future__ import print_function
 import os
 import pytest
 import subprocess
-import sys
-import tempfile
 import time
 
 import ray
-from ray.test.test_utils import run_and_get_output
-
-
-def run_string_as_driver(driver_script):
-    """Run a driver as a separate process.
-
-    Args:
-        driver_script: A string to run as a Python script.
-
-    Returns:
-        The script's output.
-    """
-    # Save the driver script as a file so we can call it using subprocess.
-    with tempfile.NamedTemporaryFile() as f:
-        f.write(driver_script.encode("ascii"))
-        f.flush()
-        out = ray.utils.decode(
-            subprocess.check_output([sys.executable, f.name]))
-    return out
-
-
-def run_string_as_driver_nonblocking(driver_script):
-    """Start a driver as a separate process and return immediately.
-
-    Args:
-        driver_script: A string to run as a Python script.
-
-    Returns:
-        A handle to the driver process.
-    """
-    # Save the driver script as a file so we can call it using subprocess. We
-    # do not delete this file because if we do then it may get removed before
-    # the Python process tries to run it.
-    with tempfile.NamedTemporaryFile(delete=False) as f:
-        f.write(driver_script.encode("ascii"))
-        f.flush()
-        return subprocess.Popen(
-            [sys.executable, f.name], stdout=subprocess.PIPE)
+from ray.test.test_utils import (run_and_get_output, run_string_as_driver,
+                                 run_string_as_driver_nonblocking)
 
 
 @pytest.fixture