Get more tests running on Windows (#6537)

* Get rid of system() calls

* Work around '/usr/share/mini' showing up on GitHub Actions (probably due to psutil truncation)

https://github.com/ray-project/ray/runs/722480047?check_suite_focus=true

* Don't check for socket max path length on Windows

* Don't check for socket existence on Windows

* Fix race condition in Windows fate-sharing

* Work around missing .exe extension for Redis tests

* Add more tests to GitHub Actions

Co-authored-by: Mehrdad <noreply@github.com>

python/ray/services.py

@@ -99,14 +99,16 @@ class ConsolePopen(subprocess.Popen):
             # CREATE_NEW_PROCESS_GROUP is used to send Ctrl+C on Windows:
             # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal
             new_pgroup = subprocess.CREATE_NEW_PROCESS_GROUP
-            flags = 0
+            flags_to_add = 0
             if ray.utils.detect_fate_sharing_support():
                 # If we don't have kernel-mode fate-sharing, then don't do this
                 # because our children need to be in out process group for
                 # the process reaper to properly terminate them.
-                flags = new_pgroup
-            kwargs.setdefault("creationflags", flags)
-            self._use_signals = (kwargs["creationflags"] & new_pgroup)
+                flags_to_add = new_pgroup
+            flags_key = "creationflags"
+            if flags_to_add:
+                kwargs[flags_key] = (kwargs.get(flags_key) or 0) | flags_to_add
+            self._use_signals = (kwargs[flags_key] & new_pgroup)
             super(ConsolePopen, self).__init__(*args, **kwargs)
 
 
@@ -503,6 +505,14 @@ def start_ray_process(command,
         if fate_share and sys.platform.startswith("linux"):
             ray.utils.set_kill_on_parent_death_linux()
 
+    win32_fate_sharing = fate_share and sys.platform == "win32"
+    # With Windows fate-sharing, we need special care:
+    # The process must be added to the job before it is allowed to execute.
+    # Otherwise, there's a race condition: the process might spawn children
+    # before the process itself is assigned to the job.
+    # After that point, its children will not be added to the job anymore.
+    CREATE_SUSPENDED = 0x00000004  # from Windows headers
+
     process = ConsolePopen(
         command,
         env=modified_env,
@@ -510,10 +520,16 @@ def start_ray_process(command,
         stdout=stdout_file,
         stderr=stderr_file,
         stdin=subprocess.PIPE if pipe_stdin else None,
-        preexec_fn=preexec_fn if sys.platform != "win32" else None)
+        preexec_fn=preexec_fn if sys.platform != "win32" else None,
+        creationflags=CREATE_SUSPENDED if win32_fate_sharing else 0)
 
-    if fate_share and sys.platform == "win32":
-        ray.utils.set_kill_child_on_death_win32(process)
+    if win32_fate_sharing:
+        try:
+            ray.utils.set_kill_child_on_death_win32(process)
+            psutil.Process(process.pid).resume()
+        except (psutil.Error, OSError):
+            process.kill()
+            raise
 
     return ProcessInfo(
         process=process,

python/ray/tests/test_metrics.py

@@ -113,9 +113,9 @@ def test_worker_stats(shutdown_only):
         ]
         for process in processes:
             # TODO(ekl) why does travis/mi end up in the process list
-            assert ("python" in process or "conda" in process
-                    or "travis" in process or "runner" in process
-                    or "ray" in process)
+            assert ("python" in process or "mini" in process
+                    or "conda" in process or "travis" in process
+                    or "runner" in process or "ray" in process)
         break
 
     # Test kill_actor.

python/ray/tests/test_tempfile.py

@@ -1,13 +1,30 @@
 import os
 import shutil
+import subprocess
+import sys
 import time
+
 import pytest
 import ray
 import ray.ray_constants as ray_constants
-import subprocess
 from ray.cluster_utils import Cluster
 
 
+def unix_socket_create_path(name):
+    unix = sys.platform != "win32"
+    return os.path.join(ray.utils.get_user_temp_dir(), name) if unix else None
+
+
+def unix_socket_verify(unix_socket):
+    if sys.platform != "win32":
+        assert os.path.exists(unix_socket), "Socket not found: " + unix_socket
+
+
+def unix_socket_delete(unix_socket):
+    unix = sys.platform != "win32"
+    return os.remove(unix_socket) if unix else None
+
+
 def test_conn_cluster():
     # plasma_store_socket_name
     with pytest.raises(Exception) as exc_info:
@@ -73,72 +90,58 @@ def test_tempdir_commandline():
 
 
 def test_tempdir_long_path():
-    temp_dir = os.path.join(ray.utils.get_user_temp_dir(), "z" * 108)
-    with pytest.raises(OSError):
-        ray.init(temp_dir=temp_dir)  # path should be too long
+    if sys.platform != "win32":
+        # Test AF_UNIX limits for sockaddr_un->sun_path on POSIX OSes
+        maxlen = 104 if sys.platform.startswith("darwin") else 108
+        temp_dir = os.path.join(ray.utils.get_user_temp_dir(), "z" * maxlen)
+        with pytest.raises(OSError):
+            ray.init(temp_dir=temp_dir)  # path should be too long
 
 
 def test_raylet_socket_name(shutdown_only):
-    ray.init(
-        raylet_socket_name=os.path.join(ray.utils.get_user_temp_dir(),
-                                        "i_am_a_temp_socket"))
-    assert os.path.exists(
-        os.path.join(ray.utils.get_user_temp_dir(),
-                     "i_am_a_temp_socket")), "Specified socket path not found."
+    sock1 = unix_socket_create_path("i_am_a_temp_socket_1")
+    ray.init(raylet_socket_name=sock1)
+    unix_socket_verify(sock1)
     ray.shutdown()
     try:
-        os.remove(
-            os.path.join(ray.utils.get_user_temp_dir(), "i_am_a_temp_socket"))
+        unix_socket_delete(sock1)
     except OSError:
         pass  # It could have been removed by Ray.
     cluster = Cluster(True)
-    cluster.add_node(
-        raylet_socket_name=os.path.join(ray.utils.get_user_temp_dir(),
-                                        "i_am_a_temp_socket_2"))
-    assert os.path.exists(
-        os.path.join(
-            ray.utils.get_user_temp_dir(),
-            "i_am_a_temp_socket_2")), "Specified socket path not found."
+    sock2 = unix_socket_create_path("i_am_a_temp_socket_2")
+    cluster.add_node(raylet_socket_name=sock2)
+    unix_socket_verify(sock2)
     cluster.shutdown()
     try:
-        os.remove(
-            os.path.join(ray.utils.get_user_temp_dir(),
-                         "i_am_a_temp_socket_2"))
+        unix_socket_delete(sock2)
     except OSError:
         pass  # It could have been removed by Ray.
 
 
 def test_temp_plasma_store_socket(shutdown_only):
-    ray.init(
-        plasma_store_socket_name=os.path.join(ray.utils.get_user_temp_dir(),
-                                              "i_am_a_temp_socket"))
-    assert os.path.exists(
-        os.path.join(ray.utils.get_user_temp_dir(),
-                     "i_am_a_temp_socket")), "Specified socket path not found."
+    sock1 = unix_socket_create_path("i_am_a_temp_socket_1")
+    ray.init(plasma_store_socket_name=sock1)
+    unix_socket_verify(sock1)
     ray.shutdown()
     try:
-        os.remove(
-            os.path.join(ray.utils.get_user_temp_dir(), "i_am_a_temp_socket"))
+        unix_socket_delete(sock1)
     except OSError:
         pass  # It could have been removed by Ray.
     cluster = Cluster(True)
-    cluster.add_node(
-        plasma_store_socket_name=os.path.join(ray.utils.get_user_temp_dir(),
-                                              "i_am_a_temp_socket_2"))
-    assert os.path.exists(
-        os.path.join(
-            ray.utils.get_user_temp_dir(),
-            "i_am_a_temp_socket_2")), "Specified socket path not found."
+    sock2 = unix_socket_create_path("i_am_a_temp_socket_2")
+    cluster.add_node(plasma_store_socket_name=sock2)
+    unix_socket_verify(sock2)
     cluster.shutdown()
     try:
-        os.remove(
-            os.path.join(ray.utils.get_user_temp_dir(),
-                         "i_am_a_temp_socket_2"))
+        unix_socket_delete(sock2)
     except OSError:
         pass  # It could have been removed by Ray.
 
 
 def test_raylet_tempfiles(shutdown_only):
+    expected_socket_files = ({"plasma_store", "raylet"}
+                             if sys.platform != "win32" else set())
+
     ray.init(num_cpus=0)
     node = ray.worker._global_node
     top_levels = set(os.listdir(node.get_session_dir_path()))
@@ -159,7 +162,7 @@ def test_raylet_tempfiles(shutdown_only):
     assert log_files.issuperset(log_files_expected)
 
     socket_files = set(os.listdir(node.get_sockets_dir_path()))
-    assert socket_files == {"plasma_store", "raylet"}
+    assert socket_files == expected_socket_files
     ray.shutdown()
 
     ray.init(num_cpus=2)
@@ -176,7 +179,7 @@ def test_raylet_tempfiles(shutdown_only):
         1 for filename in log_files if filename.startswith("worker")) == 4
 
     socket_files = set(os.listdir(node.get_sockets_dir_path()))
-    assert socket_files == {"plasma_store", "raylet"}
+    assert socket_files == expected_socket_files
 
 
 def test_tempdir_privilege(shutdown_only):
@@ -196,7 +199,6 @@ def test_session_dir_uniqueness():
 
 
 if __name__ == "__main__":
-    import sys
     # Make subprocess happy in bazel.
     os.environ["LC_ALL"] = "en_US.UTF-8"
     os.environ["LANG"] = "en_US.UTF-8"