[Core] Log monitor multidriver (#8953)

python/ray/tests/test_multi_node.py

@@ -5,13 +5,9 @@ import time
 
 import ray
 from ray.test_utils import (
-    RayTestTimeoutException,
-    run_string_as_driver,
-    run_string_as_driver_nonblocking,
-    wait_for_children_of_pid,
-    wait_for_children_of_pid_to_exit,
-    kill_process_by_name,
-)
+    RayTestTimeoutException, run_string_as_driver,
+    run_string_as_driver_nonblocking, wait_for_children_of_pid,
+    wait_for_children_of_pid_to_exit, kill_process_by_name, Semaphore)
 
 
 def test_error_isolation(call_ray_start):
@@ -634,6 +630,77 @@ print("success")
     ray.get(f.remote())
 
 
+def test_multi_driver_logging(ray_start_regular):
+    address_info = ray_start_regular
+    address = address_info["redis_address"]
+
+    # ray.init(address=address)
+    driver1_wait = Semaphore.options(name="driver1_wait").remote(value=0)
+    driver2_wait = Semaphore.options(name="driver2_wait").remote(value=0)
+    main_wait = Semaphore.options(name="main_wait").remote(value=0)
+
+    # Params are address, semaphore name, output1, output2
+    driver_script_template = """
+import ray
+import sys
+from ray.test_utils import Semaphore
+
+@ray.remote(num_cpus=0)
+def remote_print(s, file=None):
+    print(s, file=file)
+
+ray.init(address="{}")
+
+driver_wait = ray.get_actor("{}")
+main_wait = ray.get_actor("main_wait")
+
+ray.get(main_wait.release.remote())
+ray.get(driver_wait.acquire.remote())
+
+s1 = "{}"
+ray.get(remote_print.remote(s1))
+
+ray.get(main_wait.release.remote())
+ray.get(driver_wait.acquire.remote())
+
+s2 = "{}"
+ray.get(remote_print.remote(s2))
+
+ray.get(main_wait.release.remote())
+    """
+
+    p1 = run_string_as_driver_nonblocking(
+        driver_script_template.format(address, "driver1_wait", "1", "2"))
+    p2 = run_string_as_driver_nonblocking(
+        driver_script_template.format(address, "driver2_wait", "3", "4"))
+
+    ray.get(main_wait.acquire.remote())
+    ray.get(main_wait.acquire.remote())
+    # At this point both of the other drivers are fully initialized.
+
+    ray.get(driver1_wait.release.remote())
+    ray.get(driver2_wait.release.remote())
+
+    # At this point driver1 should receive '1' and driver2 '3'
+    ray.get(main_wait.acquire.remote())
+    ray.get(main_wait.acquire.remote())
+
+    ray.get(driver1_wait.release.remote())
+    ray.get(driver2_wait.release.remote())
+
+    # At this point driver1 should receive '2' and driver2 '4'
+    ray.get(main_wait.acquire.remote())
+    ray.get(main_wait.acquire.remote())
+
+    driver1_out = p1.stdout.read().decode("ascii").split("\n")
+    driver2_out = p2.stdout.read().decode("ascii").split("\n")
+
+    assert driver1_out[0][-1] == "1"
+    assert driver1_out[1][-1] == "2"
+    assert driver2_out[0][-1] == "3"
+    assert driver2_out[1][-1] == "4"
+
+
 if __name__ == "__main__":
     import pytest
     import sys