From 0f0acb8ac191bd5a91773131e94a8c7ba61aa684 Mon Sep 17 00:00:00 2001 From: alanamarzoev Date: Wed, 26 Jul 2017 00:15:50 -0700 Subject: [PATCH] CPU Time Series. (#765) Add time series of CPU utilization to web UI. --- python/ray/WebUI.ipynb | 135 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/python/ray/WebUI.ipynb b/python/ray/WebUI.ipynb index 45bf05fee..9fd7831b9 100644 --- a/python/ray/WebUI.ipynb +++ b/python/ray/WebUI.ipynb @@ -473,6 +473,141 @@ "\n", "task_completion_time_distribution()" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### CPU usage over time plot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from bokeh.layouts import gridplot\n", + "from bokeh.plotting import figure, show, helpers\n", + "from bokeh.resources import CDN\n", + "from bokeh.io import output_notebook, push_notebook\n", + "from bokeh.models import Range1d, ColumnDataSource\n", + "import numpy as np\n", + "output_notebook(resources=CDN)\n", + " \n", + "# Parse the client table to determine how many CPUs are available\n", + "num_cpus = 0 \n", + "client_table = ray.global_state.client_table()\n", + "for node_ip, client_list in client_table.items(): \n", + " for client in client_list: \n", + " if \"NumCPUs\" in client: \n", + " num_cpus += client[\"NumCPUs\"]\n", + "\n", + "def compute_utilizations(abs_earliest, abs_latest, num_tasks, tasks, num_buckets):\n", + " # Determine what the earliest and latest tasks are out of the ones that are passed in\n", + " earliest_time = time.time()\n", + " latest_time = 0\n", + " \n", + " if len(tasks) == 0:\n", + " return [], [], []\n", + " \n", + " sum_len = 0\n", + " for task_id, data in tasks.items():\n", + " latest_time = max((latest_time, data[\"store_outputs_end\"]))\n", + " earliest_time = min((earliest_time, data[\"get_arguments_start\"]))\n", + " sum_len += data[\"store_outputs_end\"] - data[\"get_arguments_start\"]\n", + " \n", + " # Add some epsilon to latest_time to ensure that the end time of the last task\n", + " # falls __within__ a bucket, and not on the edge\n", + " latest_time += 1e-6\n", + " \n", + " # Compute average CPU utilization per time bucket by summing cpu-time per bucket\n", + " bucket_time_length = (latest_time - earliest_time) / float(num_buckets)\n", + " cpu_time = [0 for _ in range(num_buckets)]\n", + " \n", + " for data in tasks.values():\n", + " task_start_time = data[\"get_arguments_start\"]\n", + " task_end_time = data[\"store_outputs_end\"]\n", + " \n", + " start_bucket = int((task_start_time - earliest_time) / bucket_time_length)\n", + " end_bucket = int((task_end_time - earliest_time) / bucket_time_length)\n", + " # Walk over each time bucket that this task intersects, adding the amount of\n", + " # time that the task intersects within each bucket\n", + " for bucket_idx in range(start_bucket, end_bucket + 1):\n", + " bucket_start_time = earliest_time + bucket_idx * bucket_time_length\n", + " bucket_end_time = earliest_time + (bucket_idx + 1) * bucket_time_length\n", + " \n", + " task_start_time_within_bucket = max(task_start_time, bucket_start_time)\n", + " task_end_time_within_bucket = min(task_end_time, bucket_end_time)\n", + " task_cpu_time_within_bucket = task_end_time_within_bucket - task_start_time_within_bucket\n", + " \n", + " cpu_time[bucket_idx] += task_cpu_time_within_bucket\n", + " \n", + " # Cpu_utilization is the average cpu utilization of the bucket, which is just\n", + " # cpu_time divided by bucket_time_length\n", + " cpu_utilization = list(map(lambda x: x / float(bucket_time_length), cpu_time))\n", + " \n", + " # Generate histogram bucket edges. Subtract out abs_earliest to get relative time\n", + " all_edges = [earliest_time - abs_earliest + i * bucket_time_length for i in range(num_buckets + 1)]\n", + " # Left edges are all but the rightmost edge, right edges are all but the leftmost edge\n", + " left_edges = all_edges[:-1]\n", + " right_edges = all_edges[1:]\n", + " \n", + " return left_edges, right_edges, cpu_utilization\n", + " \n", + "\n", + "# Update the plot based on the sliders\n", + "def plot_utilization():\n", + " # Create the Bokeh plot\n", + " time_series_fig = figure(title=\"CPU Utilization\",\n", + " tools=[\"save\", \"hover\", \"wheel_zoom\", \"box_zoom\", \"pan\"],\n", + " background_fill_color=\"#FFFFFF\", x_range=[0, 1], y_range=[0, 1])\n", + " \n", + " # Create the data source that the plot will pull from\n", + " time_series_source = ColumnDataSource(data=dict(\n", + " left=[],\n", + " right=[],\n", + " top=[]\n", + " ))\n", + " \n", + " # Plot the rectangles representing the distribution\n", + " time_series_fig.quad(left=\"left\", right=\"right\", top=\"top\", bottom=0,\n", + " source=time_series_source, fill_color=\"#B3B3B3\", line_color=\"#033649\")\n", + " \n", + " # Label the plot axes\n", + " time_series_fig.xaxis.axis_label = \"Time in seconds\"\n", + " time_series_fig.yaxis.axis_label = \"Number of CPUs used\"\n", + " \n", + " handle = show(gridplot(time_series_fig, ncols=1, plot_width=500, plot_height=500, toolbar_location=\"below\"),\n", + " notebook_handle=True)\n", + " \n", + " def update_plot(abs_earliest, abs_latest, abs_num_tasks, tasks):\n", + " num_buckets = 100\n", + " left, right, top = compute_utilizations(abs_earliest, abs_latest, abs_num_tasks, tasks, num_buckets)\n", + " \n", + " time_series_source.data = {\"left\": left, \"right\": right, \"top\": top}\n", + " \n", + " x_range = (max(0, min(left)) if len(left) else 0, max(right) if len(right) else 1)\n", + " y_range = (0, max(top) + 1 if len(top) else 1)\n", + " \n", + " # Define the axis ranges\n", + " x_range = helpers._get_range(x_range)\n", + " time_series_fig.x_range.start = x_range.start\n", + " time_series_fig.x_range.end = x_range.end\n", + " \n", + " y_range = helpers._get_range(y_range)\n", + " time_series_fig.y_range.start = y_range.start\n", + " time_series_fig.y_range.end = num_cpus\n", + " \n", + " # Push the updated data to the notebook\n", + " push_notebook(handle=handle)\n", + " \n", + " get_sliders(update_plot)\n", + "\n", + "plot_utilization()" + ] } ], "metadata": {