[Dashboard] Start the new dashboard (#10131)

* Use new dashboard if environment var RAY_USE_NEW_DASHBOARD exists; new dashboard startup

* Make fake client/build/static directory for dashboard

* Add test_dashboard.py for new dashboard

* Travis CI enable new dashboard test

* Update new dashboard

* Agent manager service

* Add agent manager

* Register agent to agent manager

* Add a new line to the end of agent_manager.cc

* Fix merge; Fix lint

* Update dashboard/agent.py

Co-authored-by: SangBin Cho <rkooo567@gmail.com>

* Update dashboard/head.py

Co-authored-by: SangBin Cho <rkooo567@gmail.com>

* Fix bug

* Add tests for dashboard

* Fix

* Remove const from Process::Kill() & Fix bugs

* Revert error check of execute_after

* Raise exception from DashboardAgent.run

* Add more tests.

* Fix compile on Linux

* Use dict comprehension instead of dict(generator)

* Fix lint

* Fix windows compile

* Fix lint

* Test Windows CI

* Revert "Test Windows CI"

This reverts commit 945e01051ec95cff5fcc1c0bc37045b46e7ad9a6.

* Fix ParseWindowsCommandLine bug

* Update src/ray/util/util.cc

Co-authored-by: Robert Nishihara <robertnishihara@gmail.com>

Co-authored-by: 刘宝 <po.lb@antfin.com>
Co-authored-by: SangBin Cho <rkooo567@gmail.com>
Co-authored-by: Robert Nishihara <robertnishihara@gmail.com>

dashboard/modules/reporter/reporter_agent.py

@@ -21,6 +21,13 @@ import psutil
 
 logger = logging.getLogger(__name__)
 
+try:
+    import gpustat.core as gpustat
+except ImportError:
+    gpustat = None
+    logger.warning(
+        "Install gpustat with 'pip install gpustat' to enable GPU monitoring.")
+
 
 def recursive_asdict(o):
     if isinstance(o, tuple) and hasattr(o, "_asdict"):
@@ -81,10 +88,35 @@ class ReporterAgent(dashboard_utils.DashboardAgentModule,
         return reporter_pb2.GetProfilingStatsReply(
             profiling_stats=profiling_stats, stdout=stdout, stderr=stderr)
 
+    async def ReportMetrics(self, request, context):
+        # TODO(sang): Process metrics here.
+        return reporter_pb2.ReportMetricsReply()
+
     @staticmethod
     def _get_cpu_percent():
         return psutil.cpu_percent()
 
+    @staticmethod
+    def _get_gpu_usage():
+        if gpustat is None:
+            return []
+        gpu_utilizations = []
+        gpus = []
+        try:
+            gpus = gpustat.new_query().gpus
+        except Exception as e:
+            logger.debug(
+                "gpustat failed to retrieve GPU information: {}".format(e))
+        for gpu in gpus:
+            # Note the keys in this dict have periods which throws
+            # off javascript so we change .s to _s
+            gpu_data = {
+                "_".join(key.split(".")): val
+                for key, val in gpu.entry.items()
+            }
+            gpu_utilizations.append(gpu_data)
+        return gpu_utilizations
+
     @staticmethod
     def _get_boot_time():
         return psutil.boot_time()
@@ -173,6 +205,7 @@ class ReporterAgent(dashboard_utils.DashboardAgentModule,
             "bootTime": self._get_boot_time(),
             "loadAvg": self._get_load_avg(),
             "disk": self._get_disk_usage(),
+            "gpus": self._get_gpu_usage(),
             "net": netstats,
             "cmdline": self._get_raylet_cmdline(),
         }

dashboard/modules/reporter/reporter_head.py

@@ -29,8 +29,8 @@ class ReportHead(dashboard_utils.DashboardHeadModule):
 
     async def _update_stubs(self, change):
         if change.new:
-            ip, port = next(iter(change.new.items()))
-            channel = aiogrpc.insecure_channel("{}:{}".format(ip, int(port)))
+            ip, ports = next(iter(change.new.items()))
+            channel = aiogrpc.insecure_channel("{}:{}".format(ip, ports[1]))
             stub = reporter_pb2_grpc.ReporterServiceStub(channel)
             self._stubs[ip] = stub
         if change.old:
@@ -77,15 +77,15 @@ class ReportHead(dashboard_utils.DashboardHeadModule):
             message="Profiling info fetched.",
             profiling_info=json.loads(profiling_stats.profiling_stats))
 
-    async def run(self):
-        p = self._dashboard_head.aioredis_client
-        mpsc = Receiver()
+    async def run(self, server):
+        aioredis_client = self._dashboard_head.aioredis_client
+        receiver = Receiver()
 
         reporter_key = "{}*".format(reporter_consts.REPORTER_PREFIX)
-        await p.psubscribe(mpsc.pattern(reporter_key))
+        await aioredis_client.psubscribe(receiver.pattern(reporter_key))
         logger.info("Subscribed to {}".format(reporter_key))
 
-        async for sender, msg in mpsc.iter():
+        async for sender, msg in receiver.iter():
             try:
                 _, data = msg
                 data = json.loads(ray.utils.decode(data))

dashboard/modules/test/test_agent.py

@@ -0,0 +1,24 @@
+import logging
+
+import aiohttp.web
+
+import ray.new_dashboard.utils as dashboard_utils
+import ray.new_dashboard.modules.test.test_utils as test_utils
+
+logger = logging.getLogger(__name__)
+routes = dashboard_utils.ClassMethodRouteTable
+
+
+class HeadAgent(dashboard_utils.DashboardAgentModule):
+    def __init__(self, dashboard_agent):
+        super().__init__(dashboard_agent)
+
+    @routes.get("/test/http_get_from_agent")
+    async def get_url(self, req) -> aiohttp.web.Response:
+        url = req.query.get("url")
+        result = await test_utils.http_get(self._dashboard_agent.http_session,
+                                           url)
+        return aiohttp.web.json_response(result)
+
+    async def run(self, server):
+        pass

dashboard/modules/test/test_head.py

@@ -0,0 +1,62 @@
+import logging
+
+import aiohttp.web
+
+import ray.new_dashboard.utils as dashboard_utils
+import ray.new_dashboard.modules.test.test_utils as test_utils
+from ray.new_dashboard.datacenter import DataSource
+
+logger = logging.getLogger(__name__)
+routes = dashboard_utils.ClassMethodRouteTable
+
+
+class TestHead(dashboard_utils.DashboardHeadModule):
+    def __init__(self, dashboard_head):
+        super().__init__(dashboard_head)
+        self._notified_agents = {}
+        DataSource.agents.signal.append(self._update_notified_agents)
+
+    async def _update_notified_agents(self, change):
+        if change.new:
+            ip, ports = next(iter(change.new.items()))
+            self._notified_agents[ip] = ports
+        if change.old:
+            ip, port = next(iter(change.old.items()))
+            self._notified_agents.pop(ip)
+
+    @routes.get("/test/dump")
+    async def dump(self, req) -> aiohttp.web.Response:
+        key = req.query.get("key")
+        if key is None:
+            all_data = {
+                k: dict(v)
+                for k, v in DataSource.__dict__.items()
+                if not k.startswith("_")
+            }
+            return await dashboard_utils.rest_response(
+                success=True,
+                message="Fetch all data from datacenter success.",
+                **all_data)
+        else:
+            data = dict(DataSource.__dict__.get(key))
+            return await dashboard_utils.rest_response(
+                success=True,
+                message="Fetch {} from datacenter success.".format(key),
+                **{key: data})
+
+    @routes.get("/test/notified_agents")
+    async def get_notified_agents(self, req) -> aiohttp.web.Response:
+        return await dashboard_utils.rest_response(
+            success=True,
+            message="Fetch notified agents success.",
+            **self._notified_agents)
+
+    @routes.get("/test/http_get")
+    async def get_url(self, req) -> aiohttp.web.Response:
+        url = req.query.get("url")
+        result = await test_utils.http_get(self._dashboard_head.http_session,
+                                           url)
+        return aiohttp.web.json_response(result)
+
+    async def run(self, server):
+        pass

dashboard/modules/test/test_utils.py

@@ -0,0 +1,11 @@
+import logging
+
+import async_timeout
+
+logger = logging.getLogger(__name__)
+
+
+async def http_get(http_session, url, timeout_seconds=60):
+    with async_timeout.timeout(timeout_seconds):
+        async with http_session.get(url) as response:
+            return await response.json()