diff --git a/BUILD.bazel b/BUILD.bazel index 61651b2f0..b59793989 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -507,7 +507,6 @@ filegroup( "python/ray/dashboard/res/main.js", "python/ray/experimental/*.py", "python/ray/internal/*.py", - "python/ray/WebUI.ipynb", "python/ray/workers/default_worker.py", ]), ) diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 3b21aacf1..c63566623 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -75,7 +75,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $PIP_CMD install -q wheel # Add the correct Python to the path and build the wheel. This is only # needed so that the installation finds the cython executable. - INCLUDE_UI=1 PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel + PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel mv dist/*.whl ../.whl/ popd done diff --git a/python/build-wheel-manylinux1.sh b/python/build-wheel-manylinux1.sh index 82db1f506..a96c3fcb0 100755 --- a/python/build-wheel-manylinux1.sh +++ b/python/build-wheel-manylinux1.sh @@ -39,7 +39,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do # Fix the numpy version because this will be the oldest numpy version we can # support. /opt/python/${PYTHON}/bin/pip install -q numpy==${NUMPY_VERSION} cython==0.29.0 - INCLUDE_UI=1 PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel + PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel # In the future, run auditwheel here. mv dist/*.whl ../.whl/ popd diff --git a/python/ray/WebUI.ipynb b/python/ray/WebUI.ipynb deleted file mode 100644 index 229366eba..000000000 --- a/python/ray/WebUI.ipynb +++ /dev/null @@ -1,97 +0,0 @@ -{ - "cells": [{ - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Ray UI\n", "\n", - "Start the UI with **Kernel -> Restart and Run All**." - ] - }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", "import ray\n", - "import ray.experimental.ui as ui\n", "\n", - "ray.init(redis_address=os.environ[\"REDIS_ADDRESS\"])" - ] - }, { - "cell_type": "markdown", - "metadata": {}, - "source": ["#### Task trace timeline."] - }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To view arrows, go to View Options and select Flow Events." - ] - }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": ["ui.task_timeline()"] - }, { - "cell_type": "markdown", - "metadata": {}, - "source": ["#### Object transfer timeline."] - }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": ["ui.object_transfer_timeline()"] - }, { - "cell_type": "markdown", - "metadata": {}, - "source": ["#### Task durations."] - }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": ["ui.task_completion_time_distribution()"] - }, { - "cell_type": "markdown", - "metadata": {}, - "source": ["#### CPU usage."] - }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": ["ui.cpu_usage()"] - }, { - "cell_type": "markdown", - "metadata": {}, - "source": ["#### Cluster usage."] - }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": ["ui.cluster_usage()"] - }], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/python/ray/experimental/ui.py b/python/ray/experimental/ui.py deleted file mode 100644 index 967336953..000000000 --- a/python/ray/experimental/ui.py +++ /dev/null @@ -1,702 +0,0 @@ -import logging -import numpy as np -import os -import pprint -import shutil -import tempfile -import time - -import ipywidgets as widgets -from IPython.display import display, IFrame, clear_output - -import ray - -logger = logging.getLogger(__name__) - - -# Instances of this class maintains keep track of whether or not a -# callback is currently executing. Since the execution of the callback -# may trigger more calls to the callback, this is used to prevent infinite -# recursions. -class _EventRecursionContextManager(object): - def __init__(self): - self.should_recurse = True - - def __enter__(self): - self.should_recurse = False - - def __exit__(self, *args): - self.should_recurse = True - - -total_time_value = "% total time" -total_tasks_value = "% total tasks" - -# Function that returns instances of sliders and handles associated events. - - -def get_sliders(update): - # Start_box value indicates the desired start point of queried window. - start_box = widgets.FloatText( - description="Start Time:", - disabled=True, - ) - - # End_box value indicates the desired end point of queried window. - end_box = widgets.FloatText( - description="End Time:", - disabled=True, - ) - - # Percentage slider. Indicates either % of total time or total tasks - # depending on what breakdown_opt is set to. - range_slider = widgets.IntRangeSlider( - value=[0, 100], - min=0, - max=100, - step=1, - description="%:", - continuous_update=False, - orientation="horizontal", - readout=True, - ) - - # Indicates the number of tasks that the user wants to be returned. Is - # disabled when the breakdown_opt value is set to total_time_value. - num_tasks_box = widgets.IntText(description="Num Tasks:", disabled=False) - - # Dropdown bar that lets the user choose between modifying % of total - # time or total number of tasks. - breakdown_opt = widgets.Dropdown( - options=[total_time_value, total_tasks_value], - value=total_tasks_value, - description="Selection Options:") - - # Display box for layout. - total_time_box = widgets.VBox([start_box, end_box]) - - # This sets the CSS style display to hide the box. - total_time_box.layout.display = 'none' - - # Initially passed in to the update_wrapper function. - INIT_EVENT = "INIT" - - # Create instance of context manager to determine whether callback is - # currently executing - out_recursion = _EventRecursionContextManager() - - def update_wrapper(event): - # Feature received a callback, but it shouldn't be executed - # because the callback was the result of a different feature - # executing its callback based on user input. - if not out_recursion.should_recurse: - return - - # Feature received a callback and it should be executed because - # the callback was the result of user input. - with out_recursion: - smallest, largest, num_tasks = ray.global_state._job_length() - diff = largest - smallest - if num_tasks != 0: - - # Describes the initial values that the slider/text box - # values should be set to. - if event == INIT_EVENT: - if breakdown_opt.value == total_tasks_value: - num_tasks_box.value = -min(10000, num_tasks) - range_slider.value = (int( - 100 - (100. * -num_tasks_box.value) / num_tasks), - 100) - else: - low, high = map(lambda x: x / 100., range_slider.value) - start_box.value = round(diff * low, 2) - end_box.value = round(diff * high, 2) - - # Event was triggered by a change in the start_box value. - elif event["owner"] == start_box: - if start_box.value > end_box.value: - start_box.value = end_box.value - elif start_box.value < 0: - start_box.value = 0 - low, high = range_slider.value - range_slider.value = (int((start_box.value * 100.) / diff), - high) - - # Event was triggered by a change in the end_box value. - elif event["owner"] == end_box: - if start_box.value > end_box.value: - end_box.value = start_box.value - elif end_box.value > diff: - end_box.value = diff - low, high = range_slider.value - range_slider.value = (low, - int((end_box.value * 100.) / diff)) - - # Event was triggered by a change in the breakdown options - # toggle. - elif event["owner"] == breakdown_opt: - if breakdown_opt.value == total_tasks_value: - start_box.disabled = True - end_box.disabled = True - num_tasks_box.disabled = False - total_time_box.layout.display = 'none' - - # Make CSS display go back to the default settings. - num_tasks_box.layout.display = None - num_tasks_box.value = min(10000, num_tasks) - range_slider.value = (int( - 100 - (100. * num_tasks_box.value) / num_tasks), - 100) - else: - start_box.disabled = False - end_box.disabled = False - num_tasks_box.disabled = True - - # Make CSS display go back to the default settings. - total_time_box.layout.display = None - num_tasks_box.layout.display = 'none' - range_slider.value = ( - int((start_box.value * 100.) / diff), - int((end_box.value * 100.) / diff)) - - # Event was triggered by a change in the range_slider - # value. - elif event["owner"] == range_slider: - low, high = map(lambda x: x / 100., range_slider.value) - if breakdown_opt.value == total_tasks_value: - old_low, old_high = event["old"] - new_low, new_high = event["new"] - if old_low != new_low: - range_slider.value = (new_low, 100) - num_tasks_box.value = ( - -(100. - new_low) / 100. * num_tasks) - else: - range_slider.value = (0, new_high) - num_tasks_box.value = new_high / 100. * num_tasks - else: - start_box.value = round(diff * low, 2) - end_box.value = round(diff * high, 2) - - # Event was triggered by a change in the num_tasks_box - # value. - elif event["owner"] == num_tasks_box: - if num_tasks_box.value > 0: - range_slider.value = ( - 0, int( - 100 * float(num_tasks_box.value) / num_tasks)) - elif num_tasks_box.value < 0: - range_slider.value = (100 + int( - 100 * float(num_tasks_box.value) / num_tasks), 100) - - # Get updated values from a slider or text box, and update the rest of - # them accordingly. - range_slider.observe(update_wrapper, names="value") - breakdown_opt.observe(update_wrapper, names="value") - start_box.observe(update_wrapper, names="value") - end_box.observe(update_wrapper, names="value") - num_tasks_box.observe(update_wrapper, names="value") - - # Initializes the sliders - update_wrapper(INIT_EVENT) - - # Display sliders and search boxes - display(breakdown_opt, - widgets.HBox([range_slider, total_time_box, num_tasks_box])) - - # Return the sliders and text boxes - return start_box, end_box, range_slider, breakdown_opt - - -def object_search_bar(): - object_search = widgets.Text( - value="", - placeholder="Object ID", - description="Search for an object:", - disabled=False) - display(object_search) - - def handle_submit(sender): - pp = pprint.PrettyPrinter() - pp.pprint(ray.global_state.object_table(object_search.value)) - - object_search.on_submit(handle_submit) - - -def task_search_bar(): - task_search = widgets.Text( - value="", - placeholder="Task ID", - description="Search for a task:", - disabled=False) - display(task_search) - - def handle_submit(sender): - pp = pprint.PrettyPrinter() - pp.pprint(ray.global_state.task_table(task_search.value)) - - task_search.on_submit(handle_submit) - - -# Hard limit on the number of tasks to return to the UI client at once -MAX_TASKS_TO_VISUALIZE = 10000 - - -# Helper function that guarantees unique and writeable temp files. -# Prevents clashes in task trace files when multiple notebooks are running. -def _get_temp_file_path(**kwargs): - temp_file = tempfile.NamedTemporaryFile( - delete=False, dir=os.getcwd(), **kwargs) - temp_file_path = temp_file.name - temp_file.close() - return os.path.relpath(temp_file_path) - - -def task_timeline(): - # Check that the trace viewer renderer file is present, and copy it to the - # current working directory if it is not present. - if not os.path.exists("trace_viewer_full.html"): - shutil.copy( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../core/src/catapult_files/trace_viewer_full.html"), - "trace_viewer_full.html") - - trace_viewer_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../core/src/catapult_files/index.html") - - html_file_path = _get_temp_file_path(suffix=".html") - json_file_path = _get_temp_file_path(suffix=".json") - - ray.global_state.chrome_tracing_dump(filename=json_file_path) - - with open(trace_viewer_path) as f: - data = f.read() - - # Replace the demo data path with our own - # https://github.com/catapult-project/catapult/blob/ - # 33a9271eb3cf5caf925293ec6a4b47c94f1ac968/tracing/bin/index.html#L107 - data = data.replace("../test_data/big_trace.json", json_file_path) - - with open(html_file_path, "w+") as f: - f.write(data) - - # Display the task trace within the Jupyter notebook - clear_output(wait=True) - logger.info("To view fullscreen, open chrome://tracing in Google Chrome " - "and load `{}`".format(os.path.abspath(json_file_path))) - display(IFrame(html_file_path, 900, 800)) - - -def object_transfer_timeline(): - # Check that the trace viewer renderer file is present, and copy it to the - # current working directory if it is not present. - if not os.path.exists("trace_viewer_full.html"): - shutil.copy( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../core/src/catapult_files/trace_viewer_full.html"), - "trace_viewer_full.html") - - trace_viewer_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../core/src/catapult_files/index.html") - - html_file_path = _get_temp_file_path(suffix=".html") - json_file_path = _get_temp_file_path(suffix=".json") - - ray.global_state.chrome_tracing_object_transfer_dump( - filename=json_file_path) - - with open(trace_viewer_path) as f: - data = f.read() - - # Replace the demo data path with our own - # https://github.com/catapult-project/catapult/blob/ - # 33a9271eb3cf5caf925293ec6a4b47c94f1ac968/tracing/bin/index.html#L107 - data = data.replace("../test_data/big_trace.json", json_file_path) - - with open(html_file_path, "w+") as f: - f.write(data) - - # Display the task trace within the Jupyter notebook - clear_output(wait=True) - logger.info("To view fullscreen, open chrome://tracing in Google Chrome " - "and load `{}`".format(os.path.abspath(json_file_path))) - display(IFrame(html_file_path, 900, 800)) - - -def task_completion_time_distribution(): - from bokeh.models import ColumnDataSource - from bokeh.layouts import gridplot - from bokeh.plotting import figure, show, helpers - from bokeh.io import output_notebook, push_notebook - from bokeh.resources import CDN - output_notebook(resources=CDN) - - # Create the Bokeh plot - p = figure( - title="Task Completion Time Distribution", - tools=["save", "hover", "wheel_zoom", "box_zoom", "pan"], - background_fill_color="#FFFFFF", - x_range=(0, 1), - y_range=(0, 1)) - - # Create the data source that the plot pulls from - source = ColumnDataSource(data={"top": [], "left": [], "right": []}) - - # Plot the histogram rectangles - p.quad( - top="top", - bottom=0, - left="left", - right="right", - source=source, - fill_color="#B3B3B3", - line_color="#033649") - - # Label the plot axes - p.xaxis.axis_label = "Duration in seconds" - p.yaxis.axis_label = "Number of tasks" - - handle = show( - gridplot( - p, - ncols=1, - plot_width=500, - plot_height=500, - toolbar_location="below"), - notebook_handle=True) - - # Function to update the plot - def task_completion_time_update(abs_earliest, abs_latest, abs_num_tasks, - tasks): - if len(tasks) == 0: - return - - # Create the distribution to plot - distr = [] - for task_id, data in tasks.items(): - distr.append(data["store_outputs_end"] - - data["get_arguments_start"]) - - # Create a histogram from the distribution - top, bin_edges = np.histogram(distr, bins="auto") - left = bin_edges[:-1] - right = bin_edges[1:] - - source.data = {"top": top, "left": left, "right": right} - - # Set the x and y ranges - x_range = (min(left) if len(left) else 0, max(right) - if len(right) else 1) - y_range = (0, max(top) + 1 if len(top) else 1) - - x_range = helpers._get_range(x_range) - p.x_range.start = x_range.start - p.x_range.end = x_range.end - - y_range = helpers._get_range(y_range) - p.y_range.start = y_range.start - p.y_range.end = y_range.end - - # Push updates to the plot - push_notebook(handle=handle) - - get_sliders(task_completion_time_update) - - -def compute_utilizations(abs_earliest, - abs_latest, - num_tasks, - tasks, - num_buckets, - use_abs_times=False): - if len(tasks) == 0: - return [], [], [] - - if use_abs_times: - earliest_time = abs_earliest - latest_time = abs_latest - else: - # Determine what the earliest and latest tasks are out of the ones - # that are passed in - earliest_time = time.time() - latest_time = 0 - for task_id, data in tasks.items(): - latest_time = max((latest_time, data["store_outputs_end"])) - earliest_time = min((earliest_time, data["get_arguments_start"])) - - # Add some epsilon to latest_time to ensure that the end time of the - # last task falls __within__ a bucket, and not on the edge - latest_time += 1e-6 - - # Compute average CPU utilization per time bucket by summing - # cpu-time per bucket - bucket_time_length = (latest_time - earliest_time) / float(num_buckets) - cpu_time = [0 for _ in range(num_buckets)] - - for data in tasks.values(): - task_start_time = data["get_arguments_start"] - task_end_time = data["store_outputs_end"] - - start_bucket = int( - (task_start_time - earliest_time) / bucket_time_length) - end_bucket = int((task_end_time - earliest_time) / bucket_time_length) - # Walk over each time bucket that this task intersects, adding the - # amount of time that the task intersects within each bucket - for bucket_idx in range(start_bucket, end_bucket + 1): - bucket_start_time = ( - (earliest_time + bucket_idx) * bucket_time_length) - bucket_end_time = ( - (earliest_time + (bucket_idx + 1)) * bucket_time_length) - - task_start_time_within_bucket = max(task_start_time, - bucket_start_time) - task_end_time_within_bucket = min(task_end_time, bucket_end_time) - task_cpu_time_within_bucket = ( - task_end_time_within_bucket - task_start_time_within_bucket) - - if bucket_idx > -1 and bucket_idx < num_buckets: - cpu_time[bucket_idx] += task_cpu_time_within_bucket - - # Cpu_utilization is the average cpu utilization of the bucket, which - # is just cpu_time divided by bucket_time_length. - cpu_utilization = list( - map(lambda x: x / float(bucket_time_length), cpu_time)) - - # Generate histogram bucket edges. Subtract out abs_earliest to get - # relative time. - all_edges = [ - earliest_time - abs_earliest + i * bucket_time_length - for i in range(num_buckets + 1) - ] - # Left edges are all but the rightmost edge, right edges are all but - # the leftmost edge. - left_edges = all_edges[:-1] - right_edges = all_edges[1:] - - return left_edges, right_edges, cpu_utilization - - -def cpu_usage(): - from bokeh.layouts import gridplot - from bokeh.plotting import figure, show, helpers - from bokeh.resources import CDN - from bokeh.io import output_notebook, push_notebook - from bokeh.models import ColumnDataSource - output_notebook(resources=CDN) - - # Parse the client table to determine how many CPUs are available - num_cpus = ray.global_state.cluster_resources()["CPU"] - - # Update the plot based on the sliders - def plot_utilization(): - # Create the Bokeh plot - time_series_fig = figure( - title="CPU Utilization", - tools=["save", "hover", "wheel_zoom", "box_zoom", "pan"], - background_fill_color="#FFFFFF", - x_range=[0, 1], - y_range=[0, 1]) - - # Create the data source that the plot will pull from - time_series_source = ColumnDataSource(data={ - 'left': [], - 'right': [], - 'top': [] - }) - - # Plot the rectangles representing the distribution - time_series_fig.quad( - left="left", - right="right", - top="top", - bottom=0, - source=time_series_source, - fill_color="#B3B3B3", - line_color="#033649") - - # Label the plot axes - time_series_fig.xaxis.axis_label = "Time in seconds" - time_series_fig.yaxis.axis_label = "Number of CPUs used" - - handle = show( - gridplot( - time_series_fig, - ncols=1, - plot_width=500, - plot_height=500, - toolbar_location="below"), - notebook_handle=True) - - def update_plot(abs_earliest, abs_latest, abs_num_tasks, tasks): - num_buckets = 100 - left, right, top = compute_utilizations( - abs_earliest, abs_latest, abs_num_tasks, tasks, num_buckets) - - time_series_source.data = { - "left": left, - "right": right, - "top": top - } - - x_range = (max(0, min(left)) if len(left) else 0, max(right) - if len(right) else 1) - y_range = (0, max(top) + 1 if len(top) else 1) - - # Define the axis ranges - x_range = helpers._get_range(x_range) - time_series_fig.x_range.start = x_range.start - time_series_fig.x_range.end = x_range.end - - y_range = helpers._get_range(y_range) - time_series_fig.y_range.start = y_range.start - time_series_fig.y_range.end = num_cpus - - # Push the updated data to the notebook - push_notebook(handle=handle) - - get_sliders(update_plot) - - plot_utilization() - - -# Function to create the cluster usage "heat map" -def cluster_usage(): - from bokeh.io import show, output_notebook, push_notebook - from bokeh.resources import CDN - from bokeh.plotting import figure - from bokeh.models import ( - ColumnDataSource, - HoverTool, - LinearColorMapper, - BasicTicker, - ColorBar, - ) - output_notebook(resources=CDN) - - # Initial values - source = ColumnDataSource( - data={ - "node_ip_address": ['127.0.0.1'], - "time": ['0.5'], - "num_tasks": ['1'], - "length": [1] - }) - - # Define the color schema - colors = [ - "#75968f", "#a5bab7", "#c9d9d3", "#e2e2e2", "#dfccce", "#ddb7b1", - "#cc7878", "#933b41", "#550b1d" - ] - mapper = LinearColorMapper(palette=colors, low=0, high=2) - - TOOLS = "hover, save, xpan, box_zoom, reset, xwheel_zoom" - - # Create the plot - p = figure( - title="Cluster Usage", - y_range=list(set(source.data['node_ip_address'])), - x_axis_location="above", - plot_width=900, - plot_height=500, - tools=TOOLS, - toolbar_location='below') - - # Format the plot axes - p.grid.grid_line_color = None - p.axis.axis_line_color = None - p.axis.major_tick_line_color = None - p.axis.major_label_text_font_size = "10pt" - p.axis.major_label_standoff = 0 - p.xaxis.major_label_orientation = np.pi / 3 - - # Plot rectangles - p.rect( - x="time", - y="node_ip_address", - width="length", - height=1, - source=source, - fill_color={ - "field": "num_tasks", - "transform": mapper - }, - line_color=None) - - # Add legend to the side of the plot - color_bar = ColorBar( - color_mapper=mapper, - major_label_text_font_size="8pt", - ticker=BasicTicker(desired_num_ticks=len(colors)), - label_standoff=6, - border_line_color=None, - location=(0, 0)) - p.add_layout(color_bar, "right") - - # Define hover tool - p.select_one(HoverTool).tooltips = [("Node IP Address", - "@node_ip_address"), - ("Number of tasks running", - "@num_tasks"), ("Time", "@time")] - - # Define the axis labels - p.xaxis.axis_label = "Time in seconds" - p.yaxis.axis_label = "Node IP Address" - handle = show(p, notebook_handle=True) - workers = ray.global_state.workers() - - # Function to update the heat map - def heat_map_update(abs_earliest, abs_latest, abs_num_tasks, tasks): - if len(tasks) == 0: - return - - earliest = time.time() - latest = 0 - - node_to_tasks = {} - # Determine which task has the earlest start time out of the ones - # passed into the update function - for task_id, data in tasks.items(): - if data["score"] > latest: - latest = data["score"] - if data["score"] < earliest: - earliest = data["score"] - worker_id = data["worker_id"] - node_ip = workers[worker_id]["node_ip_address"] - if node_ip not in node_to_tasks: - node_to_tasks[node_ip] = {} - node_to_tasks[node_ip][task_id] = data - - nodes = [] - times = [] - lengths = [] - num_tasks = [] - - for node_ip, task_dict in node_to_tasks.items(): - left, right, top = compute_utilizations( - earliest, latest, abs_num_tasks, task_dict, 100, True) - for (l, r, t) in zip(left, right, top): - nodes.append(node_ip) - times.append((l + r) / 2) - lengths.append(r - l) - num_tasks.append(t) - - # Set the y range of the plot to be the node IP addresses - p.y_range.factors = list(set(nodes)) - - mapper.low = min(min(num_tasks), 0) - mapper.high = max(max(num_tasks), 1) - - # Update plot with new data based on slider and text box values - source.data = { - "node_ip_address": nodes, - "time": times, - "num_tasks": num_tasks, - "length": lengths - } - - push_notebook(handle=handle) - - get_sliders(heat_map_update) diff --git a/python/ray/node.py b/python/ray/node.py index 5f3d89be6..c3d0e0866 100644 --- a/python/ray/node.py +++ b/python/ray/node.py @@ -319,22 +319,6 @@ class Node(object): redis_client = self.create_redis_client() redis_client.hmset("webui", {"url": self._webui_url}) - def start_ui(self): - """Start the web UI.""" - stdout_file, stderr_file = self.new_log_files("webui") - notebook_name = self._make_inc_temp( - suffix=".ipynb", prefix="ray_ui", directory_name=self._temp_dir) - _, process_info = ray.services.start_ui( - self._redis_address, - notebook_name, - stdout_file=stdout_file, - stderr_file=stderr_file) - assert ray_constants.PROCESS_TYPE_WEB_UI not in self.all_processes - if process_info is not None: - self.all_processes[ray_constants.PROCESS_TYPE_WEB_UI] = [ - process_info - ] - def start_plasma_store(self): """Start the plasma store.""" assert self._plasma_store_socket_name is None diff --git a/python/ray/parameter.py b/python/ray/parameter.py index 016ddcef4..33c1322a2 100644 --- a/python/ray/parameter.py +++ b/python/ray/parameter.py @@ -56,7 +56,7 @@ class RayParams(object): huge_pages: Boolean flag indicating whether to start the Object Store with hugetlbfs support. Requires plasma_directory. include_webui: Boolean flag indicating whether to start the web - UI, which is a Jupyter notebook. + UI, which displays the status of the Ray cluster. logging_level: Logging level, default will be logging.INFO. logging_format: Logging format, default contains a timestamp, filename, line number, and message. See ray_constants.py. diff --git a/python/ray/services.py b/python/ray/services.py index 42b28581b..4bfa4719c 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -10,7 +10,6 @@ import multiprocessing import os import random import resource -import shutil import socket import subprocess import sys @@ -943,75 +942,6 @@ def start_dashboard(redis_address, return dashboard_url, process_info -def start_ui(redis_address, notebook_name, stdout_file=None, stderr_file=None): - """Start a UI process. - - Args: - redis_address: The address of the primary Redis shard. - notebook_name: The destination of the notebook file. - stdout_file: A file handle opened for writing to redirect stdout to. If - no redirection should happen, then this should be None. - stderr_file: A file handle opened for writing to redirect stderr to. If - no redirection should happen, then this should be None. - - Returns: - A tuple of the web UI url and ProcessInfo for the process that was - started. - """ - - port = 8888 - while True: - try: - port_test_socket = socket.socket() - port_test_socket.bind(("127.0.0.1", port)) - port_test_socket.close() - break - except socket.error: - port += 1 - - notebook_filepath = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "WebUI.ipynb") - # We copy the notebook file so that the original doesn't get modified by - # the user. - shutil.copy(notebook_filepath, notebook_name) - - new_notebook_directory = os.path.dirname(notebook_name) - # We generate the token used for authentication ourselves to avoid - # querying the jupyter server. - token = ray.utils.decode(binascii.hexlify(os.urandom(24))) - # The --ip=0.0.0.0 flag is intended to enable connecting to a notebook - # running within a docker container (from the outside). - command = [ - "jupyter", "notebook", "--no-browser", "--port={}".format(port), - "--ip=0.0.0.0", "--NotebookApp.iopub_data_rate_limit=10000000000", - "--NotebookApp.open_browser=False", - "--NotebookApp.token={}".format(token) - ] - # If the user is root, add the --allow-root flag. - if os.geteuid() == 0: - command.append("--allow-root") - - try: - process_info = start_ray_process( - command, - ray_constants.PROCESS_TYPE_WEB_UI, - env_updates={"REDIS_ADDRESS": redis_address}, - cwd=new_notebook_directory, - stdout_file=stdout_file, - stderr_file=stderr_file) - except Exception: - logger.warning("Failed to start the UI, you may need to run " - "'pip install jupyter'.") - else: - webui_url = ("http://localhost:{}/notebooks/{}?token={}".format( - port, os.path.basename(notebook_name), token)) - print("\n" + "=" * 70) - print("View the web UI at {}".format(webui_url)) - print("=" * 70 + "\n") - return webui_url, process_info - return None, None - - def check_and_update_resources(num_cpus, num_gpus, resources): """Sanity check a resource dictionary and add sensible defaults. diff --git a/python/ray/worker.py b/python/ray/worker.py index c7ee4e4d7..3938d9256 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -1342,7 +1342,7 @@ def init(redis_address=None, huge_pages: Boolean flag indicating whether to start the Object Store with hugetlbfs support. Requires plasma_directory. include_webui: Boolean flag indicating whether to start the web - UI, which is a Jupyter notebook. + UI, which displays the status of the Ray cluster. driver_id: The ID of driver. configure_logging: True if allow the logging cofiguration here. Otherwise, the users may want to configure it by their own. diff --git a/python/setup.py b/python/setup.py index 1b36b0ffa..921517207 100644 --- a/python/setup.py +++ b/python/setup.py @@ -24,9 +24,8 @@ ray_files = [ "ray/core/src/ray/gcs/redis_module/libray_redis_module.so", "ray/core/src/plasma/plasma_store_server", "ray/_raylet.so", "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet", - "ray/WebUI.ipynb", "ray/dashboard/dashboard.py", - "ray/dashboard/index.html", "ray/dashboard/res/main.css", - "ray/dashboard/res/main.js" + "ray/dashboard/dashboard.py", "ray/dashboard/index.html", + "ray/dashboard/res/main.css", "ray/dashboard/res/main.js" ] # These are the directories where automatically generated Python flatbuffer @@ -38,11 +37,6 @@ generated_python_directories = [ optional_ray_files = [] -ray_ui_files = [ - "ray/core/src/catapult_files/index.html", - "ray/core/src/catapult_files/trace_viewer_full.html" -] - ray_autoscaler_files = [ "ray/autoscaler/aws/example-full.yaml", "ray/autoscaler/gcp/example-full.yaml", @@ -56,13 +50,6 @@ if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on": "ray/core/src/credis/redis/src/redis-server" ] -# The UI files are mandatory if the INCLUDE_UI environment variable equals 1. -# Otherwise, they are optional. -if "INCLUDE_UI" in os.environ and os.environ["INCLUDE_UI"] == "1": - ray_files += ray_ui_files -else: - optional_ray_files += ray_ui_files - optional_ray_files += ray_autoscaler_files extras = { diff --git a/thirdparty/scripts/build_ui.sh b/thirdparty/scripts/build_ui.sh deleted file mode 100755 index eeab6ca91..000000000 --- a/thirdparty/scripts/build_ui.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -set -x - -# Cause the script to exit if a single command fails. -set -e - -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../ - -CATAPULT_COMMIT=18cd334755701cf0c3b90b7172126c686d2eb787 -CATAPULT_HOME=$TP_DIR/pkg/catapult -VULCANIZE_BIN=$CATAPULT_HOME/tracing/bin/vulcanize_trace_viewer - -CATAPULT_FILES=$TP_DIR/../build/src/catapult_files - -# This is where we will copy the files that need to be packaged with the wheels. -mkdir -p $CATAPULT_FILES - -if [[ "$INCLUDE_UI" == "0" ]]; then - # Let installation continue without building the UI. - exit 0 -fi - -if ! type python2 > /dev/null; then - echo "cannot properly set up UI without a python2 executable" - if [[ "$INCLUDE_UI" == "1" ]]; then - # Since the UI is explicitly supposed to be included, fail here. - exit 1 - else - # Let installation continue without building the UI. - exit 0 - fi -fi - -# Download catapult and use it to autogenerate some static html if it isn't -# already present. -if [[ ! -d $CATAPULT_HOME ]]; then - echo "setting up catapult" - # The git clone command seems to fail in Travis, so retry up to 20 times. - for COUNT in {1..20}; do - # Attempt to git clone catapult and break from the retry loop if it succeeds. - git clone -q https://github.com/ray-project/catapult.git $CATAPULT_HOME && break - # If none of the retries succeeded at getting boost, then fail. - if [[ $COUNT == 20 ]]; then - exit 1 - fi - done -fi - -REBUILD=off - -# Check out the appropriate commit from catapult. -pushd $CATAPULT_HOME -if [ "$CATAPULT_COMMIT" != `git rev-parse HEAD` ]; then - git fetch origin master - git checkout $CATAPULT_COMMIT - REBUILD=on -fi -popd - -# If the autogenerated catapult files aren't present, then generate them. -if [[ ! -f $CATAPULT_FILES/index.html || "$REBUILD" == "on" ]]; then - python2 $VULCANIZE_BIN --config chrome --output $CATAPULT_FILES/trace_viewer_full.html - cp $CATAPULT_HOME/tracing/bin/index.html $CATAPULT_FILES/index.html -fi diff --git a/thirdparty/scripts/setup.sh b/thirdparty/scripts/setup.sh index da283bd3b..4327a668d 100755 --- a/thirdparty/scripts/setup.sh +++ b/thirdparty/scripts/setup.sh @@ -57,12 +57,6 @@ bash "$TP_SCRIPT_DIR/build_credis.sh" #RAY_BUILD_JAVA=$RAY_BUILD_JAVA \ #bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE -############################################## -# catapult -############################################## -# Clone catapult and build the static HTML needed for the UI. -bash "$TP_SCRIPT_DIR/build_ui.sh" - ############################################## # rDSN (optional) ##############################################