Make test_utils.py use pipes to avoid file access conflicts on Windows (#9072)

Co-authored-by: Mehrdad <noreply@github.com>

python/ray/test_utils.py

@@ -5,7 +5,6 @@ import fnmatch
 import os
 import subprocess
 import sys
-import tempfile
 import time
 import socket
 
@@ -104,13 +103,17 @@ def run_string_as_driver(driver_script):
     Returns:
         The script's output.
     """
-    # Save the driver script as a file so we can call it using subprocess.
-    with tempfile.NamedTemporaryFile() as f:
-        f.write(driver_script.encode("ascii"))
-        f.flush()
-        out = ray.utils.decode(
-            subprocess.check_output(
-                [sys.executable, f.name], stderr=subprocess.STDOUT))
+    proc = subprocess.Popen(
+        [sys.executable, "-"],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+    with proc:
+        output = proc.communicate(driver_script.encode("ascii"))[0]
+        if proc.returncode:
+            raise subprocess.CalledProcessError(proc.returncode, proc.args,
+                                                output, proc.stderr)
+        out = ray.utils.decode(output)
     return out
 
 
@@ -123,16 +126,21 @@ def run_string_as_driver_nonblocking(driver_script):
     Returns:
         A handle to the driver process.
     """
-    # Save the driver script as a file so we can call it using subprocess. We
-    # do not delete this file because if we do then it may get removed before
-    # the Python process tries to run it.
-    with tempfile.NamedTemporaryFile(delete=False) as f:
-        f.write(driver_script.encode("ascii"))
-        f.flush()
-        return subprocess.Popen(
-            [sys.executable, f.name],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
+    script = "; ".join([
+        "import sys",
+        "script = sys.stdin.read()",
+        "sys.stdin.close()",
+        "del sys",
+        "exec(\"del script\\n\" + script)",
+    ])
+    proc = subprocess.Popen(
+        [sys.executable, "-c", script],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    proc.stdin.write(driver_script.encode("ascii"))
+    proc.stdin.close()
+    return proc
 
 
 def flat_errors():