[Core] Multi-tenancy: Reject worker registration if job has finished (#10569)

python/ray/tests/test_multi_tenancy.py

@@ -9,7 +9,8 @@ import pytest
 import ray
 import ray.test_utils
 from ray.core.generated import node_manager_pb2, node_manager_pb2_grpc
-from ray.test_utils import wait_for_condition, run_string_as_driver_nonblocking
+from ray.test_utils import (wait_for_condition, run_string_as_driver,
+                            run_string_as_driver_nonblocking)
 
 
 def get_num_workers():
@@ -211,5 +212,34 @@ def test_worker_capping_run_chained_tasks(shutdown_only):
     assert get_num_workers() == 2
 
 
+def test_worker_registration_failure_after_driver_exit(shutdown_only):
+    info = ray.init(num_cpus=1, _system_config={"enable_multi_tenancy": True})
+
+    driver_code = """
+import ray
+import time
+
+
+ray.init(address="{}")
+
+@ray.remote
+def foo():
+    pass
+
+[foo.remote() for _ in range(100)]
+
+ray.shutdown()
+    """.format(info["redis_address"])
+
+    before = get_num_workers()
+    assert before == 1
+
+    run_string_as_driver(driver_code)
+
+    # wait for a while to let workers register
+    time.sleep(2)
+    wait_for_condition(lambda: get_num_workers() == before, timeout=10)
+
+
 if __name__ == "__main__":
     sys.exit(pytest.main(["-v", __file__]))