From e19e2c6284c7bd485a07ca8d53305c4a9f84cb73 Mon Sep 17 00:00:00 2001 From: Wapaul1 Date: Tue, 5 Sep 2017 23:31:44 -0700 Subject: [PATCH] Print jupyter notebook token when starting web UI. (#887) * User now only needs to copy url to get to notebook * Fixed duplicate code * Added function to print url * Added exception for calling function on worker * Stored webui url in Redis * Fix linting and simplify code. * Now uses 24 bytes hex token * Fixed python 3 compatibility * Fix linting and python 3 compat * Added comment explaining generating the token. * Removed newline * Small fixes. * Fixed jenkins failure * Rebased and changed formatting * Revert "changed formatting" This reverts commit 226510cf0cdcaab9cf42ad30bd9588a963683592. --- python/ray/__init__.py | 7 ++++--- python/ray/services.py | 29 ++++++++++++++++++----------- python/ray/worker.py | 34 ++++++++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/python/ray/__init__.py b/python/ray/__init__.py index 8e16b1c1d..73fd70336 100644 --- a/python/ray/__init__.py +++ b/python/ray/__init__.py @@ -42,7 +42,7 @@ except ImportError as e: from ray.worker import (register_class, error_info, init, connect, disconnect, get, put, wait, remote, log_event, log_span, - flush_log, get_gpu_ids) # noqa: E402 + flush_log, get_gpu_ids, get_webui_url) # noqa: E402 from ray.worker import (SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, SILENT_MODE) # noqa: E402 from ray.worker import global_state # noqa: E402 @@ -56,8 +56,9 @@ __version__ = "0.2.0" __all__ = ["register_class", "error_info", "init", "connect", "disconnect", "get", "put", "wait", "remote", "log_event", "log_span", - "flush_log", "actor", "get_gpu_ids", "SCRIPT_MODE", "WORKER_MODE", - "PYTHON_MODE", "SILENT_MODE", "global_state", "__version__"] + "flush_log", "actor", "get_gpu_ids", "get_webui_url", + "SCRIPT_MODE", "WORKER_MODE", "PYTHON_MODE", "SILENT_MODE", + "global_state", "__version__"] import ctypes # noqa: E402 # Windows only diff --git a/python/ray/services.py b/python/ray/services.py index f3278cadc..82ffe4f12 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import binascii from collections import namedtuple, OrderedDict import os import psutil @@ -492,10 +493,14 @@ def start_ui(redis_address, stdout_file=None, stderr_file=None, cleanup=True): port += 1 new_env = os.environ.copy() new_env["REDIS_ADDRESS"] = redis_address + # We generate the token used for authentication ourselves to avoid + # querying the jupyter server. + token = binascii.hexlify(os.urandom(24)).decode("ascii") command = ["jupyter", "notebook", "--no-browser", "--port={}".format(port), "--NotebookApp.iopub_data_rate_limit=10000000000", - "--NotebookApp.open_browser=False"] + "--NotebookApp.open_browser=False", + "--NotebookApp.token={}".format(token)] try: ui_process = subprocess.Popen(command, env=new_env, cwd=new_notebook_directory, @@ -506,13 +511,12 @@ def start_ui(redis_address, stdout_file=None, stderr_file=None, cleanup=True): else: if cleanup: all_processes[PROCESS_TYPE_WEB_UI].append(ui_process) - - print() - print("=" * 70) - print("View the web UI at http://localhost:{}/notebooks/ray_ui{}.ipynb" - .format(port, random_ui_id)) - print("=" * 70) - print() + webui_url = ("http://localhost:{}/notebooks/ray_ui{}.ipynb?token={}" + .format(port, random_ui_id, token)) + print("\n" + "=" * 70) + print("View the web UI at {}".format(webui_url)) + print("=" * 70 + "\n") + return webui_url def start_local_scheduler(redis_address, @@ -1004,9 +1008,12 @@ def start_ray_processes(address_info=None, if include_webui: ui_stdout_file, ui_stderr_file = new_log_files( "webui", redirect_output=True) - start_ui(redis_address, stdout_file=ui_stdout_file, - stderr_file=ui_stderr_file, cleanup=cleanup) - + address_info["webui_url"] = start_ui(redis_address, + stdout_file=ui_stdout_file, + stderr_file=ui_stderr_file, + cleanup=cleanup) + else: + address_info["webui_url"] = "" # Return the addresses of the relevant processes. return address_info diff --git a/python/ray/worker.py b/python/ray/worker.py index 303f0255a..b50e1d985 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -861,6 +861,30 @@ def get_gpu_ids(): return global_worker.local_scheduler_client.gpu_ids() +def _webui_url_helper(client): + """Parsing for getting the url of the web UI. + + Args: + client: A redis client to use to query the primary Redis shard. + + Returns: + The URL of the web UI as a string. + """ + result = client.hmget("webui", "url")[0] + return result.decode("ascii") if result is not None else result + + +def get_webui_url(): + """Get the URL to access the web UI. + + Note that the URL does not specify which node the web UI is on. + + Returns: + The URL of the web UI as a string. + """ + return _webui_url_helper(global_worker.redis_client) + + global_worker = Worker() """Worker: The global Worker object for this worker process. @@ -1059,7 +1083,9 @@ def get_address_info_from_redis_helper(redis_address, node_ip_address): client_info = {"node_ip_address": node_ip_address, "redis_address": redis_address, "object_store_addresses": object_store_addresses, - "local_scheduler_socket_names": scheduler_names} + "local_scheduler_socket_names": scheduler_names, + # Web UI should be running. + "webui_url": _webui_url_helper(redis_client)} return client_info @@ -1239,7 +1265,8 @@ def _init(address_info=None, "manager_socket_name": ( address_info["object_store_addresses"][0].manager_name), "local_scheduler_socket_name": ( - address_info["local_scheduler_socket_names"][0])} + address_info["local_scheduler_socket_names"][0]), + "webui_url": address_info["webui_url"]} connect(driver_address_info, object_id_seed=object_id_seed, mode=driver_mode, worker=global_worker, actor_id=NIL_ACTOR_ID) return address_info @@ -1608,6 +1635,7 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker, # Set the node IP address. worker.node_ip_address = info["node_ip_address"] worker.redis_address = info["redis_address"] + # Create a Redis client. redis_ip_address, redis_port = info["redis_address"].split(":") worker.redis_client = redis.StrictRedis(host=redis_ip_address, @@ -1652,6 +1680,8 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker, driver_info["name"] = (main.__file__ if hasattr(main, "__file__") else "INTERACTIVE MODE") worker.redis_client.hmset(b"Drivers:" + worker.worker_id, driver_info) + if not worker.redis_client.exists("webui"): + worker.redis_client.hmset("webui", {"url": info["webui_url"]}) is_worker = False elif mode == WORKER_MODE: # Register the worker with Redis.