Add test to test raylet client connection when raylet crashes. (#3518)

test/failure_test.py

@@ -8,9 +8,11 @@ import os
 import ray
 import sys
 import tempfile
+import threading
 import time
 
 import ray.ray_constants as ray_constants
+from ray.utils import _random_string
 import pytest
 
 
@@ -611,3 +613,18 @@ def test_warning_for_dead_node(ray_start_two_nodes):
     }
 
     assert client_ids == warning_client_ids
+
+
+def test_raylet_crash_when_get(ray_start_regular):
+    nonexistent_id = ray.ObjectID(_random_string())
+
+    def sleep_to_kill_raylet():
+        # Don't kill raylet before default workers get connected.
+        time.sleep(2)
+        ray.services.all_processes[ray.services.PROCESS_TYPE_RAYLET][0].kill()
+
+    thread = threading.Thread(target=sleep_to_kill_raylet)
+    thread.start()
+    with pytest.raises(Exception, match=r".*raylet client may be closed.*"):
+        ray.get(nonexistent_id)
+    thread.join()