diff --git a/dashboard/modules/reporter/reporter_head.py b/dashboard/modules/reporter/reporter_head.py index 49d577bca..2fdd001d4 100644 --- a/dashboard/modules/reporter/reporter_head.py +++ b/dashboard/modules/reporter/reporter_head.py @@ -12,6 +12,8 @@ import ray.new_dashboard.modules.reporter.reporter_consts as reporter_consts import ray.new_dashboard.utils as dashboard_utils import ray._private.services import ray.utils +from ray.autoscaler._private.util import (DEBUG_AUTOSCALING_STATUS, + DEBUG_AUTOSCALING_ERROR) from ray.core.generated import reporter_pb2 from ray.core.generated import reporter_pb2_grpc from ray.new_dashboard.datacenter import DataSource @@ -97,6 +99,29 @@ class ReportHead(dashboard_utils.DashboardHeadModule): **self._ray_config, ) + @routes.get("/api/cluster_status") + async def get_cluster_status(self, req): + """Returns status information about the cluster. + + Currently contains two fields: + autoscaling_status (str): a status message from the autoscaler. + autoscaling_error (str): an error message from the autoscaler if + anything has gone wrong during autoscaling. + + These fields are both read from the GCS, it's expected that the + autoscaler writes them there. + """ + + aioredis_client = self._dashboard_head.aioredis_client + status = await aioredis_client.hget(DEBUG_AUTOSCALING_STATUS, "value") + error = await aioredis_client.hget(DEBUG_AUTOSCALING_ERROR, "value") + return dashboard_utils.rest_response( + success=True, + message="Got cluster status.", + autoscaling_status=status.decode() if status else None, + autoscaling_error=error.decode() if error else None, + ) + async def run(self, server): aioredis_client = self._dashboard_head.aioredis_client receiver = Receiver() diff --git a/dashboard/tests/test_dashboard.py b/dashboard/tests/test_dashboard.py index 47d5926fa..07e30985c 100644 --- a/dashboard/tests/test_dashboard.py +++ b/dashboard/tests/test_dashboard.py @@ -20,6 +20,8 @@ from ray.test_utils import ( wait_until_server_available, run_string_as_driver, ) +from ray.autoscaler._private.util import (DEBUG_AUTOSCALING_STATUS, + DEBUG_AUTOSCALING_ERROR) import ray.new_dashboard.consts as dashboard_consts import ray.new_dashboard.utils as dashboard_utils import ray.new_dashboard.modules @@ -442,5 +444,54 @@ def test_aiohttp_cache(enable_test_module, ray_start_with_dashboard): assert len(data[0]) == 2 +def test_get_cluster_status(ray_start_with_dashboard): + assert (wait_until_server_available(ray_start_with_dashboard["webui_url"]) + is True) + address_info = ray_start_with_dashboard + webui_url = address_info["webui_url"] + webui_url = format_web_url(webui_url) + + # Check that the cluster_status endpoint works without the underlying data + # from the GCS, but returns nothing. + last_exception = None + for _ in range(5): + try: + response = requests.get(f"{webui_url}/api/cluster_status") + response.raise_for_status() + assert response.json()["result"] + assert "autoscalingStatus" in response.json()["data"] + assert response.json()["data"]["autoscalingStatus"] is None + assert "autoscalingError" in response.json()["data"] + assert response.json()["data"]["autoscalingError"] is None + break + except requests.RequestException as e: + last_exception = e + time.sleep(1) + else: + raise last_exception + + # Populate the GCS field, check that the data is returned from the + # endpoint. + address = address_info["redis_address"] + address = address.split(":") + assert len(address) == 2 + + client = redis.StrictRedis( + host=address[0], + port=int(address[1]), + password=ray_constants.REDIS_DEFAULT_PASSWORD) + + client.hset(DEBUG_AUTOSCALING_STATUS, "value", "hello") + client.hset(DEBUG_AUTOSCALING_ERROR, "value", "world") + + response = requests.get(f"{webui_url}/api/cluster_status") + response.raise_for_status() + assert response.json()["result"] + assert "autoscalingStatus" in response.json()["data"] + assert response.json()["data"]["autoscalingStatus"] == "hello" + assert "autoscalingError" in response.json()["data"] + assert response.json()["data"]["autoscalingError"] == "world" + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__]))