[Dashboard] Add dashboard multi-node churn test (#11768)

dashboard/BUILD

@@ -13,7 +13,19 @@ py_library(
 py_test_run_all_subdirectory(
     size = "small",
     include = ["**/test*.py"],
-    exclude = ["modules/test/**"],
+    exclude = ["modules/test/**", "modules/stats_collector/tests/test_stats_collector.py", "tests/test_dashboard.py"],
     extra_srcs = [],
     tags = ["exclusive"],
 )
+
+py_test(
+    name = "test_stats_collector",
+    size = "medium",
+    srcs = ["modules/stats_collector/tests/test_stats_collector.py"],
+)
+
+py_test(
+    name = "test_dashboard",
+    size = "medium",
+    srcs = ["tests/test_dashboard.py"],
+)

dashboard/modules/stats_collector/tests/test_stats_collector.py

@@ -4,9 +4,12 @@ import logging
 import requests
 import time
 import traceback
-
+import random
 import pytest
 import ray
+import threading
+from datetime import datetime, timedelta
+from ray.cluster_utils import Cluster
 from ray.new_dashboard.tests.conftest import *  # noqa
 from ray.test_utils import (format_web_url, wait_until_server_available,
                             wait_for_condition,
@@ -182,7 +185,7 @@ def test_get_all_node_details(disable_aiohttp_cache, ray_start_with_dashboard):
     }], indirect=True)
 def test_multi_nodes_info(enable_test_module, disable_aiohttp_cache,
                           ray_start_cluster_head):
-    cluster = ray_start_cluster_head
+    cluster: Cluster = ray_start_cluster_head
     assert (wait_until_server_available(cluster.webui_url) is True)
     webui_url = cluster.webui_url
     webui_url = format_web_url(webui_url)
@@ -214,7 +217,53 @@ def test_multi_nodes_info(enable_test_module, disable_aiohttp_cache,
             logger.info(ex)
             return False
 
-    wait_for_condition(_check_nodes, timeout=10)
+    wait_for_condition(_check_nodes, timeout=15)
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head", [{
+        "include_dashboard": True
+    }], indirect=True)
+def test_multi_node_churn(enable_test_module, disable_aiohttp_cache,
+                          ray_start_cluster_head):
+    cluster: Cluster = ray_start_cluster_head
+    assert (wait_until_server_available(cluster.webui_url) is True)
+    webui_url = format_web_url(cluster.webui_url)
+
+    def cluster_chaos_monkey():
+        worker_nodes = []
+        while True:
+            time.sleep(5)
+            if len(worker_nodes) < 2:
+                worker_nodes.append(cluster.add_node())
+                continue
+            should_add_node = random.randint(0, 1)
+            if should_add_node:
+                worker_nodes.append(cluster.add_node())
+            else:
+                node_index = random.randrange(0, len(worker_nodes))
+                node_to_remove = worker_nodes.pop(node_index)
+                cluster.remove_node(node_to_remove)
+
+    def get_index():
+        resp = requests.get(webui_url)
+        resp.raise_for_status()
+
+    def get_nodes():
+        resp = requests.get(webui_url + "/nodes?view=summary")
+        resp.raise_for_status()
+        summary = resp.json()
+        assert summary["result"] is True, summary["msg"]
+        assert summary["data"]["summary"]
+
+    t = threading.Thread(target=cluster_chaos_monkey, daemon=True)
+    t.start()
+
+    t_st = datetime.now()
+    duration = timedelta(seconds=60)
+    while datetime.now() < t_st + duration:
+        get_index()
+        time.sleep(2)
 
 
 @pytest.mark.parametrize(

dashboard/tests/test_dashboard.py

@@ -80,7 +80,7 @@ def test_basic(ray_start_with_dashboard):
         0]
     dashboard_proc = psutil.Process(dashboard_proc_info.process.pid)
     assert dashboard_proc.status() in [
-        psutil.STATUS_RUNNING, psutil.STATUS_SLEEPING
+        psutil.STATUS_RUNNING, psutil.STATUS_SLEEPING, psutil.STATUS_DISK_SLEEP
     ]
     raylet_proc_info = all_processes[ray_constants.PROCESS_TYPE_RAYLET][0]
     raylet_proc = psutil.Process(raylet_proc_info.process.pid)