From 3a0206a1f40a8be1205559919b9e6bbc60c62f0c Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Thu, 16 Nov 2017 00:42:28 -0800 Subject: [PATCH] [tune] Parallel Coordinate Visualization Notebook (#1218) --- .../ParallelCoordinatesVisualization.ipynb | 128 ++++++++++++++++++ python/ray/tune/visual_utils.py | 82 +++++++++++ 2 files changed, 210 insertions(+) create mode 100644 python/ray/tune/ParallelCoordinatesVisualization.ipynb create mode 100644 python/ray/tune/visual_utils.py diff --git a/python/ray/tune/ParallelCoordinatesVisualization.ipynb b/python/ray/tune/ParallelCoordinatesVisualization.ipynb new file mode 100644 index 000000000..6bb922268 --- /dev/null +++ b/python/ray/tune/ParallelCoordinatesVisualization.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tune Visualization" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to visualize results, please install `plotly` with the following command:\n", + "\n", + " `pip install plotly`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from ray.tune.visual_utils import load_results_to_df, generate_plotly_dim_dict\n", + "import plotly\n", + "import plotly.graph_objs as go\n", + "plotly.offline.init_notebook_mode(connected=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Specify the directory where all your results are in the variable `RESULTS_DIR`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "RESULTS_DIR = \"/tmp/ray/\"\n", + "df = load_results_to_df(RESULTS_DIR)\n", + "[key for key in df]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Choose the fields you wish to visualize over in `GOOD_FIELDS`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "scrolled": true + }, + "outputs": [], + "source": [ + "GOOD_FIELDS = ['experiment_id',\n", + " 'num_sgd_iter',\n", + " 'timesteps_total',\n", + " 'episode_len_mean',\n", + " 'episode_reward_mean']\n", + "\n", + "visualization_df = df[GOOD_FIELDS]\n", + "visualization_df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Enjoy.\n", + "\n", + "Documentation for this Plotly visualization can be found here: https://plot.ly/python/parallel-coordinates-plot/" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "data = [go.Parcoords(\n", + " line = dict(color = 'blue'),\n", + " dimensions = [generate_plotly_dim_dict(visualization_df, field) \n", + " for field in visualization_df])\n", + "]\n", + "\n", + "plotly.offline.iplot(data)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/ray/tune/visual_utils.py b/python/ray/tune/visual_utils.py new file mode 100644 index 000000000..833079942 --- /dev/null +++ b/python/ray/tune/visual_utils.py @@ -0,0 +1,82 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import pandas as pd +from pandas.api.types import is_string_dtype, is_numeric_dtype + +import os +import os.path as osp +import numpy as np +import json + + +def _flatten_dict(dt): + while any(type(v) is dict for v in dt.values()): + remove = [] + add = {} + for key, value in dt.items(): + if type(value) is dict: + for subkey, v in value.items(): + add[":".join([key, subkey])] = v + remove.append(key) + dt.update(add) + for k in remove: + del dt[k] + return dt + + +def _parse_results(res_path): + res_dict = {} + try: + with open(res_path) as f: + # Get last line in file + for line in f: + pass + res_dict = _flatten_dict(json.loads(line.strip())) + except Exception as e: + print("Importing %s failed...Perhaps empty?" % res_path) + return res_dict + + +def _parse_configs(cfg_path): + try: + with open(cfg_path) as f: + cfg_dict = _flatten_dict(json.load(f)) + except Exception as e: + print(e) + return cfg_dict + + +def _resolve(directory, result_fname): + resultp = osp.join(directory, result_fname) + res_dict = _parse_results(resultp) + cfgp = osp.join(directory, "config.json") + cfg_dict = _parse_configs(cfgp) + cfg_dict.update(res_dict) + return cfg_dict + + +def load_results_to_df(directory, result_name="result.json"): + exp_directories = [dirpath for dirpath, dirs, files in os.walk(directory) + for f in files if f == result_name] + data = [_resolve(directory, result_name) for directory in exp_directories] + return pd.DataFrame(data) + + +def generate_plotly_dim_dict(df, field): + dim_dict = {} + dim_dict["label"] = field + column = df[field] + if is_numeric_dtype(column): + dim_dict["values"] = column + elif is_string_dtype(column): + texts = column.unique() + dim_dict["values"] = [np.argwhere(texts == x).flatten()[0] + for x in column] + dim_dict["tickvals"] = list(range(len(texts))) + dim_dict["ticktext"] = texts + else: + raise Exception("Unidentifiable Type") + + return dim_dict