Render tasks that are not schedulable on the dashboard. (#7034)

python/ray/tests/test_metrics.py

@@ -11,7 +11,8 @@ from ray.core.generated import node_manager_pb2
 from ray.core.generated import node_manager_pb2_grpc
 from ray.core.generated import reporter_pb2
 from ray.core.generated import reporter_pb2_grpc
-from ray.test_utils import RayTestTimeoutException
+from ray.test_utils import (RayTestTimeoutException,
+                            wait_until_succeeded_without_exception)
 
 
 def test_worker_stats(shutdown_only):
@@ -249,6 +250,86 @@ def test_raylet_info_endpoint(shutdown_only):
         time.sleep(1)
 
 
+def test_raylet_infeasible_tasks(shutdown_only):
+    """
+    This test creates an actor that requires 5 GPUs
+    but a ray cluster only has 3 GPUs. As a result,
+    the new actor should be an infeasible actor.
+    """
+    addresses = ray.init(num_gpus=3)
+
+    @ray.remote(num_gpus=5)
+    class ActorRequiringGPU:
+        def __init__(self):
+            pass
+
+    ActorRequiringGPU.remote()
+
+    def test_infeasible_actor(ray_addresses):
+        webui_url = ray_addresses["webui_url"].replace("localhost",
+                                                       "http://127.0.0.1")
+        raylet_info = requests.get(webui_url + "/api/raylet_info").json()
+        actor_info = raylet_info["result"]["actors"]
+        assert len(actor_info) == 1
+
+        _, infeasible_actor_info = actor_info.popitem()
+        assert infeasible_actor_info["state"] == -1
+        assert infeasible_actor_info["invalidStateType"] == "infeasibleActor"
+
+    assert (wait_until_succeeded_without_exception(
+        test_infeasible_actor,
+        (AssertionError, requests.exceptions.ConnectionError),
+        addresses,
+        timeout_ms=30000,
+        retry_interval_ms=1000) is True)
+
+
+def test_raylet_pending_tasks(shutdown_only):
+    # Make sure to specify num_cpus. Otherwise, the test can be broken
+    # when the number of cores is less than the number of spawned actors.
+    addresses = ray.init(num_gpus=3, num_cpus=4)
+
+    @ray.remote(num_gpus=1)
+    class ActorRequiringGPU:
+        def __init__(self):
+            pass
+
+    @ray.remote
+    class ParentActor:
+        def __init__(self):
+            self.a = [ActorRequiringGPU.remote() for i in range(4)]
+
+    ParentActor.remote()
+
+    def test_pending_actor(ray_addresses):
+        webui_url = ray_addresses["webui_url"].replace("localhost",
+                                                       "http://127.0.0.1")
+        raylet_info = requests.get(webui_url + "/api/raylet_info").json()
+        actor_info = raylet_info["result"]["actors"]
+        assert len(actor_info) == 1
+        _, infeasible_actor_info = actor_info.popitem()
+
+        # Verify there are 4 spawned actors.
+        children = infeasible_actor_info["children"]
+        assert len(children) == 4
+
+        pending_actor_detected = 0
+        for child_id, child in children.items():
+            if ("invalidStateType" in child
+                    and child["invalidStateType"] == "pendingActor"):
+                pending_actor_detected += 1
+        # 4 GPUActors are spawned although there are only 3 GPUs.
+        # One actor should be in the pending state.
+        assert pending_actor_detected == 1
+
+    assert (wait_until_succeeded_without_exception(
+        test_pending_actor,
+        (AssertionError, requests.exceptions.ConnectionError),
+        addresses,
+        timeout_ms=30000,
+        retry_interval_ms=1000) is True)
+
+
 @pytest.mark.skipif(
     os.environ.get("TRAVIS") is None,
     reason="This test requires password-less sudo due to py-spy requirement.")