[Dashboard] Start the new dashboard (#10131)

* Use new dashboard if environment var RAY_USE_NEW_DASHBOARD exists; new dashboard startup

* Make fake client/build/static directory for dashboard

* Add test_dashboard.py for new dashboard

* Travis CI enable new dashboard test

* Update new dashboard

* Agent manager service

* Add agent manager

* Register agent to agent manager

* Add a new line to the end of agent_manager.cc

* Fix merge; Fix lint

* Update dashboard/agent.py

Co-authored-by: SangBin Cho <rkooo567@gmail.com>

* Update dashboard/head.py

Co-authored-by: SangBin Cho <rkooo567@gmail.com>

* Fix bug

* Add tests for dashboard

* Fix

* Remove const from Process::Kill() & Fix bugs

* Revert error check of execute_after

* Raise exception from DashboardAgent.run

* Add more tests.

* Fix compile on Linux

* Use dict comprehension instead of dict(generator)

* Fix lint

* Fix windows compile

* Fix lint

* Test Windows CI

* Revert "Test Windows CI"

This reverts commit 945e01051ec95cff5fcc1c0bc37045b46e7ad9a6.

* Fix ParseWindowsCommandLine bug

* Update src/ray/util/util.cc

Co-authored-by: Robert Nishihara <robertnishihara@gmail.com>

Co-authored-by: 刘宝 <po.lb@antfin.com>
Co-authored-by: SangBin Cho <rkooo567@gmail.com>
Co-authored-by: Robert Nishihara <robertnishihara@gmail.com>

python/ray/node.py

@@ -615,8 +615,11 @@ class Node:
                 if we fail to start the dashboard. Otherwise it will print
                 a warning if we fail to start the dashboard.
         """
-        stdout_file, stderr_file = self.get_log_file_handles(
-            "dashboard", unique=True)
+        if "RAY_USE_NEW_DASHBOARD" in os.environ:
+            stdout_file, stderr_file = None, None
+        else:
+            stdout_file, stderr_file = self.get_log_file_handles(
+                "dashboard", unique=True)
         self._webui_url, process_info = ray.services.start_dashboard(
             require_dashboard,
             self._ray_params.dashboard_host,
@@ -797,7 +800,8 @@ class Node:
 
         self.start_plasma_store()
         self.start_raylet()
-        self.start_reporter()
+        if "RAY_USE_NEW_DASHBOARD" not in os.environ:
+            self.start_reporter()
 
         if self._ray_params.include_log_monitor:
             self.start_log_monitor()

python/ray/services.py

@@ -1160,8 +1160,14 @@ def start_dashboard(require_dashboard,
             raise ValueError(
                 f"The given dashboard port {port} is already in use")
 
+    if "RAY_USE_NEW_DASHBOARD" in os.environ:
+        dashboard_dir = "new_dashboard"
+    else:
+        dashboard_dir = "dashboard"
+
     dashboard_filepath = os.path.join(
-        os.path.dirname(os.path.abspath(__file__)), "dashboard/dashboard.py")
+        os.path.dirname(os.path.abspath(__file__)), dashboard_dir,
+        "dashboard.py")
     command = [
         sys.executable,
         "-u",
@@ -1398,6 +1404,23 @@ def start_raylet(redis_address,
         start_worker_command.append(
             f"--object-spilling-config={json.dumps(object_spilling_config)}")
 
+    # Create agent command
+    agent_command = [
+        sys.executable,
+        "-u",
+        os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "new_dashboard/agent.py"),
+        "--redis-address={}".format(redis_address),
+        "--node-manager-port={}".format(node_manager_port),
+        "--object-store-name={}".format(plasma_store_name),
+        "--raylet-name={}".format(raylet_name),
+        "--temp-dir={}".format(temp_dir),
+    ]
+
+    if redis_password is not None and len(redis_password) != 0:
+        agent_command.append("--redis-password={}".format(redis_password))
+
     command = [
         RAYLET_EXECUTABLE,
         f"--raylet_socket_name={raylet_name}",
@@ -1424,6 +1447,9 @@ def start_raylet(redis_address,
     if start_initial_python_workers_for_first_job:
         command.append("--num_initial_python_workers_for_first_job={}".format(
             resource_spec.num_cpus))
+    if "RAY_USE_NEW_DASHBOARD" in os.environ:
+        command.append("--agent_command={}".format(
+            subprocess.list2cmdline(agent_command)))
     if config.get("plasma_store_as_thread"):
         # command related to the plasma store
         plasma_directory, object_store_memory = determine_plasma_store_config(

python/setup.py

@@ -133,6 +133,7 @@ extras["all"] = list(set(chain.from_iterable(extras.values())))
 # the change in the matching section of requirements.txt
 install_requires = [
     "aiohttp",
+    "aiohttp_cors",
     "aioredis",
     "click >= 7.0",
     "colorama",
@@ -408,7 +409,7 @@ def api_main(program, *args):
             nonlocal result
             if excinfo[1].errno != errno.ENOENT:
                 msg = excinfo[1].strerror
-                logger.error("cannot remove {}: {}" % (path, msg))
+                logger.error("cannot remove {}: {}".format(path, msg))
                 result = 1
 
         for subdir in CLEANABLE_SUBDIRS: