[dashboard] Add /api/cluster_status endpoint (#11456)

dashboard/modules/reporter/reporter_head.py

@@ -12,6 +12,8 @@ import ray.new_dashboard.modules.reporter.reporter_consts as reporter_consts
 import ray.new_dashboard.utils as dashboard_utils
 import ray._private.services
 import ray.utils
+from ray.autoscaler._private.util import (DEBUG_AUTOSCALING_STATUS,
+                                          DEBUG_AUTOSCALING_ERROR)
 from ray.core.generated import reporter_pb2
 from ray.core.generated import reporter_pb2_grpc
 from ray.new_dashboard.datacenter import DataSource
@@ -97,6 +99,29 @@ class ReportHead(dashboard_utils.DashboardHeadModule):
             **self._ray_config,
         )
 
+    @routes.get("/api/cluster_status")
+    async def get_cluster_status(self, req):
+        """Returns status information about the cluster.
+
+        Currently contains two fields:
+            autoscaling_status (str): a status message from the autoscaler.
+            autoscaling_error (str): an error message from the autoscaler if
+                anything has gone wrong during autoscaling.
+
+        These fields are both read from the GCS, it's expected that the
+        autoscaler writes them there.
+        """
+
+        aioredis_client = self._dashboard_head.aioredis_client
+        status = await aioredis_client.hget(DEBUG_AUTOSCALING_STATUS, "value")
+        error = await aioredis_client.hget(DEBUG_AUTOSCALING_ERROR, "value")
+        return dashboard_utils.rest_response(
+            success=True,
+            message="Got cluster status.",
+            autoscaling_status=status.decode() if status else None,
+            autoscaling_error=error.decode() if error else None,
+        )
+
     async def run(self, server):
         aioredis_client = self._dashboard_head.aioredis_client
         receiver = Receiver()

dashboard/tests/test_dashboard.py

@@ -20,6 +20,8 @@ from ray.test_utils import (
     wait_until_server_available,
     run_string_as_driver,
 )
+from ray.autoscaler._private.util import (DEBUG_AUTOSCALING_STATUS,
+                                          DEBUG_AUTOSCALING_ERROR)
 import ray.new_dashboard.consts as dashboard_consts
 import ray.new_dashboard.utils as dashboard_utils
 import ray.new_dashboard.modules
@@ -442,5 +444,54 @@ def test_aiohttp_cache(enable_test_module, ray_start_with_dashboard):
     assert len(data[0]) == 2
 
 
+def test_get_cluster_status(ray_start_with_dashboard):
+    assert (wait_until_server_available(ray_start_with_dashboard["webui_url"])
+            is True)
+    address_info = ray_start_with_dashboard
+    webui_url = address_info["webui_url"]
+    webui_url = format_web_url(webui_url)
+
+    # Check that the cluster_status endpoint works without the underlying data
+    # from the GCS, but returns nothing.
+    last_exception = None
+    for _ in range(5):
+        try:
+            response = requests.get(f"{webui_url}/api/cluster_status")
+            response.raise_for_status()
+            assert response.json()["result"]
+            assert "autoscalingStatus" in response.json()["data"]
+            assert response.json()["data"]["autoscalingStatus"] is None
+            assert "autoscalingError" in response.json()["data"]
+            assert response.json()["data"]["autoscalingError"] is None
+            break
+        except requests.RequestException as e:
+            last_exception = e
+            time.sleep(1)
+    else:
+        raise last_exception
+
+    # Populate the GCS field, check that the data is returned from the
+    # endpoint.
+    address = address_info["redis_address"]
+    address = address.split(":")
+    assert len(address) == 2
+
+    client = redis.StrictRedis(
+        host=address[0],
+        port=int(address[1]),
+        password=ray_constants.REDIS_DEFAULT_PASSWORD)
+
+    client.hset(DEBUG_AUTOSCALING_STATUS, "value", "hello")
+    client.hset(DEBUG_AUTOSCALING_ERROR, "value", "world")
+
+    response = requests.get(f"{webui_url}/api/cluster_status")
+    response.raise_for_status()
+    assert response.json()["result"]
+    assert "autoscalingStatus" in response.json()["data"]
+    assert response.json()["data"]["autoscalingStatus"] == "hello"
+    assert "autoscalingError" in response.json()["data"]
+    assert response.json()["data"]["autoscalingError"] == "world"
+
+
 if __name__ == "__main__":
     sys.exit(pytest.main(["-v", __file__]))