[Dashboard] Add sorted columns and TensorBoard to Tune tab (#7140)

python/ray/dashboard/client/src/api.ts

@@ -219,8 +219,8 @@ export interface TuneTrial {
   status: string;
   trial_id: string;
   job_id: string;
-  params: { [key: string]: string };
-  metrics: { [key: string]: string };
+  params: { [key: string]: string | number };
+  metrics: { [key: string]: string | number };
 }
 
 export interface TuneJobResponse {

python/ray/dashboard/client/src/pages/dashboard/Dashboard.tsx

@@ -13,7 +13,7 @@ import LogicalView from "./logical-view/LogicalView";
 import NodeInfo from "./node-info/NodeInfo";
 import RayConfig from "./ray-config/RayConfig";
 import { dashboardActions } from "./state";
-import Tune from "./Tune";
+import Tune from "./tune/Tune";
 
 const styles = (theme: Theme) =>
   createStyles({

python/ray/dashboard/client/src/pages/dashboard/Tune.tsx

@@ -1,162 +0,0 @@
-import { Theme } from "@material-ui/core/styles/createMuiTheme";
-import createStyles from "@material-ui/core/styles/createStyles";
-import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles";
-import React from "react";
-import { connect } from "react-redux";
-import { getTuneInfo } from "../../api";
-import { StoreState } from "../../store";
-import { dashboardActions } from "./state";
-import Typography from "@material-ui/core/Typography";
-import WarningRoundedIcon from "@material-ui/icons/WarningRounded";
-import Table from "@material-ui/core/Table";
-import TableBody from "@material-ui/core/TableBody";
-import TableCell from "@material-ui/core/TableCell";
-import TableHead from "@material-ui/core/TableHead";
-import TableRow from "@material-ui/core/TableRow";
-
-const styles = (theme: Theme) =>
-  createStyles({
-    root: {
-      padding: theme.spacing(2),
-      "& > :not(:first-child)": {
-        marginTop: theme.spacing(2)
-      }
-    },
-    table: {
-      marginTop: theme.spacing(1)
-    },
-    cell: {
-      padding: theme.spacing(1),
-      textAlign: "center",
-      "&:last-child": {
-        paddingRight: theme.spacing(1)
-      }
-    },
-    warning: {
-      fontSize: "0.8125rem",
-      marginBottom: theme.spacing(2)
-    },
-    warningIcon: {
-      fontSize: "1.25em",
-      verticalAlign: "text-bottom"
-    }
-  });
-
-const mapStateToProps = (state: StoreState) => ({
-  tuneInfo: state.dashboard.tuneInfo
-});
-
-const mapDispatchToProps = dashboardActions;
-
-class Tune extends React.Component<
-  WithStyles<typeof styles> &
-    ReturnType<typeof mapStateToProps> &
-    typeof mapDispatchToProps
-> {
-  timeout: number = 0;
-
-  refreshTuneInfo = async () => {
-    try {
-      const [tuneInfo] = await Promise.all([getTuneInfo()]);
-      this.props.setTuneInfo({ tuneInfo });
-    } catch (error) {
-      this.props.setError(error.toString());
-    } finally {
-      this.timeout = window.setTimeout(this.refreshTuneInfo, 1000);
-    }
-  };
-
-  async componentDidMount() {
-    await this.refreshTuneInfo();
-  }
-
-  async componentWillUnmount() {
-    window.clearTimeout(this.timeout);
-  }
-
-  render() {
-    const { classes, tuneInfo } = this.props;
-
-    if (
-      tuneInfo === null ||
-      Object.keys(tuneInfo["trial_records"]).length === 0
-    ) {
-      return null;
-    }
-
-    const firstTrial = Object.keys(tuneInfo["trial_records"])[0];
-    let paramNames: string[] = [];
-    if (tuneInfo !== null) {
-      const paramsDict = tuneInfo["trial_records"][firstTrial]["params"];
-      paramNames = Object.keys(paramsDict).filter(k => k !== "args");
-    }
-
-    const metricNames = Object.keys(
-      tuneInfo["trial_records"][firstTrial]["metrics"]
-    );
-
-    return (
-      <div className={classes.root}>
-        <Typography className={classes.warning} color="textSecondary">
-          <WarningRoundedIcon className={classes.warningIcon} /> Note: This tab
-          is experimental.
-        </Typography>
-        <Table className={classes.table}>
-          <TableHead>
-            <TableRow>
-              <TableCell className={classes.cell}>Trial ID</TableCell>
-              <TableCell className={classes.cell}>Job ID</TableCell>
-              <TableCell className={classes.cell}>Start Time</TableCell>
-              {paramNames.map((value, index) => (
-                <TableCell className={classes.cell} key={value}>
-                  {value}
-                </TableCell>
-              ))}
-              <TableCell className={classes.cell}>Status</TableCell>
-              {metricNames.map((value, index) => (
-                <TableCell className={classes.cell} key={value}>
-                  {value}
-                </TableCell>
-              ))}
-            </TableRow>
-          </TableHead>
-          <TableBody>
-            {tuneInfo != null &&
-              Object.keys(tuneInfo["trial_records"]).map(key => (
-                <TableRow key={key}>
-                  <TableCell className={classes.cell}>
-                    {tuneInfo["trial_records"][key]["trial_id"]}
-                  </TableCell>
-                  <TableCell className={classes.cell}>
-                    {tuneInfo["trial_records"][key]["job_id"]}
-                  </TableCell>
-                  <TableCell className={classes.cell}>
-                    {tuneInfo["trial_records"][key]["start_time"]}
-                  </TableCell>
-                  {paramNames.map((value, index) => (
-                    <TableCell className={classes.cell} key={value}>
-                      {tuneInfo["trial_records"][key]["params"][value]}
-                    </TableCell>
-                  ))}
-                  <TableCell className={classes.cell}>
-                    {tuneInfo["trial_records"][key]["status"]}
-                  </TableCell>
-                  {tuneInfo["trial_records"][key]["metrics"] &&
-                    metricNames.map((value, index) => (
-                      <TableCell className={classes.cell} key={value}>
-                        {tuneInfo["trial_records"][key]["metrics"][value]}
-                      </TableCell>
-                    ))}
-                </TableRow>
-              ))}
-          </TableBody>
-        </Table>
-      </div>
-    );
-  }
-}
-
-export default connect(
-  mapStateToProps,
-  mapDispatchToProps
-)(withStyles(styles)(Tune));

python/ray/dashboard/client/src/pages/dashboard/state.ts

@@ -52,13 +52,8 @@ const slice = createSlice({
       state.rayletInfo = action.payload.rayletInfo;
       state.lastUpdatedAt = Date.now();
     },
-    setTuneInfo: (
-      state,
-      action: PayloadAction<{
-        tuneInfo: TuneJobResponse;
-      }>
-    ) => {
-      state.tuneInfo = action.payload.tuneInfo;
+    setTuneInfo: (state, action: PayloadAction<TuneJobResponse>) => {
+      state.tuneInfo = action.payload;
       state.lastUpdatedAt = Date.now();
     },
     setTuneAvailability: (

python/ray/dashboard/client/src/pages/dashboard/tune/Tune.tsx

@@ -0,0 +1,119 @@
+import { Theme } from "@material-ui/core/styles/createMuiTheme";
+import createStyles from "@material-ui/core/styles/createStyles";
+import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles";
+import React from "react";
+import { connect } from "react-redux";
+import { getTuneInfo } from "../../../api";
+import { StoreState } from "../../../store";
+import { dashboardActions } from "../state";
+import Typography from "@material-ui/core/Typography";
+import WarningRoundedIcon from "@material-ui/icons/WarningRounded";
+import Tab from "@material-ui/core/Tab";
+import Tabs from "@material-ui/core/Tabs";
+import TuneTable from "./TuneTable";
+import TuneTensorBoard from "./TuneTensorBoard";
+
+const styles = (theme: Theme) =>
+  createStyles({
+    root: {
+      backgroundColor: theme.palette.background.paper
+    },
+    tabs: {
+      borderBottomColor: theme.palette.divider,
+      borderBottomStyle: "solid",
+      borderBottomWidth: 1
+    },
+    warning: {
+      fontSize: "0.8125rem"
+    },
+    warningIcon: {
+      fontSize: "1.25em",
+      verticalAlign: "text-bottom"
+    }
+  });
+
+const mapStateToProps = (state: StoreState) => ({
+  tuneInfo: state.dashboard.tuneInfo
+});
+
+const mapDispatchToProps = dashboardActions;
+
+interface State {
+  tabIndex: number;
+}
+
+class Tune extends React.Component<
+  WithStyles<typeof styles> &
+    ReturnType<typeof mapStateToProps> &
+    typeof mapDispatchToProps,
+  State
+> {
+  timeout: number = 0;
+
+  state: State = {
+    tabIndex: 0
+  };
+
+  refreshTuneInfo = async () => {
+    try {
+      const tuneInfo = await getTuneInfo();
+      this.props.setTuneInfo(tuneInfo);
+    } catch (error) {
+      this.props.setError(error.toString());
+    } finally {
+      this.timeout = window.setTimeout(this.refreshTuneInfo, 1000);
+    }
+  };
+
+  async componentDidMount() {
+    await this.refreshTuneInfo();
+  }
+
+  async componentWillUnmount() {
+    window.clearTimeout(this.timeout);
+  }
+
+  handleTabChange = (event: React.ChangeEvent<{}>, value: number) => {
+    this.setState({
+      tabIndex: value
+    });
+  };
+
+  render() {
+    const { classes } = this.props;
+
+    const { tabIndex } = this.state;
+
+    const tabs = [
+      { label: "Table", component: TuneTable },
+      { label: "TensorBoard", component: TuneTensorBoard }
+    ];
+
+    const SelectedComponent = tabs[tabIndex].component;
+    return (
+      <div className={classes.root}>
+        <Typography className={classes.warning} color="textSecondary">
+          <WarningRoundedIcon className={classes.warningIcon} /> Note: This tab
+          is experimental.
+        </Typography>
+        <Tabs
+          className={classes.tabs}
+          indicatorColor="primary"
+          onChange={this.handleTabChange}
+          textColor="primary"
+          value={tabIndex}
+        >
+          {tabs.map(({ label }) => (
+            <Tab key={label} label={label} />
+          ))}
+        </Tabs>
+        <SelectedComponent />
+      </div>
+    );
+  }
+}
+
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(withStyles(styles)(Tune));

python/ray/dashboard/client/src/pages/dashboard/tune/TuneTable.tsx

@@ -0,0 +1,241 @@
+import { Theme } from "@material-ui/core/styles/createMuiTheme";
+import createStyles from "@material-ui/core/styles/createStyles";
+import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles";
+import React from "react";
+import { connect } from "react-redux";
+import { StoreState } from "../../../store";
+import { dashboardActions } from "../state";
+import Table from "@material-ui/core/Table";
+import TableBody from "@material-ui/core/TableBody";
+import TableCell from "@material-ui/core/TableCell";
+import TableHead from "@material-ui/core/TableHead";
+import TableRow from "@material-ui/core/TableRow";
+import TableSortLabel from "@material-ui/core/TableSortLabel";
+import { TuneTrial } from "../../../api";
+
+const styles = (theme: Theme) =>
+  createStyles({
+    root: {
+      padding: theme.spacing(2),
+      "& > :not(:first-child)": {
+        marginTop: theme.spacing(2)
+      }
+    },
+    table: {
+      marginTop: theme.spacing(1)
+    },
+    cell: {
+      padding: theme.spacing(1),
+      textAlign: "right",
+      "&:last-child": {
+        paddingRight: theme.spacing(1)
+      }
+    }
+  });
+
+const mapStateToProps = (state: StoreState) => ({
+  tuneInfo: state.dashboard.tuneInfo
+});
+
+interface State {
+  metricParamColumn: string;
+  ascending: boolean;
+  sortedColumn: keyof TuneTrial | undefined;
+}
+
+const mapDispatchToProps = dashboardActions;
+
+class TuneTable extends React.Component<
+  WithStyles<typeof styles> &
+    ReturnType<typeof mapStateToProps> &
+    typeof mapDispatchToProps,
+  State
+> {
+  timeout: number = 0;
+
+  state: State = {
+    sortedColumn: undefined,
+    ascending: true,
+    metricParamColumn: ""
+  };
+
+  onColumnClick = (column: keyof TuneTrial, metricParamColumn?: string) => {
+    let ascending = this.state.ascending;
+    if (column === this.state.sortedColumn) {
+      ascending = !ascending;
+    } else {
+      ascending = true;
+    }
+    this.setState({
+      sortedColumn: column,
+      ascending: ascending
+    });
+
+    if (metricParamColumn) {
+      this.setState({
+        metricParamColumn: metricParamColumn
+      });
+    }
+  };
+
+  /**
+   * Replaces all underscores with spaces and capitalizes all words
+   * in str
+   */
+  humanize = (str: string) => {
+    var i,
+      frags = str.split("_");
+    for (i = 0; i < frags.length; i++) {
+      frags[i] = frags[i].charAt(0).toUpperCase() + frags[i].slice(1);
+    }
+    return frags.join(" ");
+  };
+
+  sortedCell = (name: keyof TuneTrial, chosenMetricParam?: string) => {
+    const { tuneInfo, classes } = this.props;
+    const { sortedColumn, ascending, metricParamColumn } = this.state;
+    let label: "desc" | "asc" = "asc";
+
+    if (name === sortedColumn && !ascending) {
+      label = "desc";
+    }
+
+    if (tuneInfo === null) {
+      return;
+    }
+
+    let onClick = () => this.onColumnClick(name);
+    if (chosenMetricParam) {
+      onClick = () => this.onColumnClick(name, chosenMetricParam);
+    }
+
+    let active = false;
+    let key: string = name;
+    if (chosenMetricParam) {
+      key = chosenMetricParam;
+      active = chosenMetricParam === metricParamColumn && sortedColumn === name;
+    } else {
+      active = name === sortedColumn;
+    }
+
+    return (
+      <TableCell className={classes.cell} key={key} onClick={onClick}>
+        <TableSortLabel active={active} direction={label} />
+        {chosenMetricParam
+          ? this.humanize(chosenMetricParam)
+          : this.humanize(name)}
+      </TableCell>
+    );
+  };
+
+  sortedTrialRecords = () => {
+    const { tuneInfo } = this.props;
+    const { sortedColumn, ascending, metricParamColumn } = this.state;
+
+    if (
+      tuneInfo === null ||
+      Object.keys(tuneInfo["trial_records"]).length === 0
+    ) {
+      return null;
+    }
+
+    const trialDetails = Object.values(tuneInfo["trial_records"]);
+
+    if (!sortedColumn) {
+      return trialDetails;
+    }
+
+    let getAttribute = (trial: TuneTrial) => trial[sortedColumn!];
+    if (sortedColumn === "metrics" || sortedColumn === "params") {
+      getAttribute = (trial: TuneTrial) =>
+        trial[sortedColumn!][metricParamColumn];
+    }
+
+    if (sortedColumn) {
+      if (ascending) {
+        trialDetails.sort((a, b) =>
+          getAttribute(a) > getAttribute(b) ? 1 : -1
+        );
+      } else if (!ascending) {
+        trialDetails.sort((a, b) =>
+          getAttribute(a) < getAttribute(b) ? 1 : -1
+        );
+      }
+    }
+
+    return trialDetails;
+  };
+
+  render() {
+    const { classes, tuneInfo } = this.props;
+
+    if (
+      tuneInfo === null ||
+      Object.keys(tuneInfo["trial_records"]).length === 0
+    ) {
+      return null;
+    }
+
+    const firstTrial = Object.keys(tuneInfo["trial_records"])[0];
+    const paramsDict = tuneInfo["trial_records"][firstTrial]["params"];
+    const paramNames = Object.keys(paramsDict).filter(k => k !== "args");
+
+    const metricNames = Object.keys(
+      tuneInfo["trial_records"][firstTrial]["metrics"]
+    );
+
+    const trialDetails = this.sortedTrialRecords();
+
+    return (
+      <div className={classes.root}>
+        <Table className={classes.table}>
+          <TableHead>
+            <TableRow>
+              {this.sortedCell("trial_id")}
+              {this.sortedCell("job_id")}
+              {this.sortedCell("start_time")}
+              {paramNames.map(value => this.sortedCell("params", value))}
+              {this.sortedCell("status")}
+              {metricNames.map(value => this.sortedCell("metrics", value))}
+            </TableRow>
+          </TableHead>
+          <TableBody>
+            {trialDetails !== null &&
+              trialDetails.map((trial, index) => (
+                <TableRow key={index}>
+                  <TableCell className={classes.cell}>
+                    {trial["trial_id"]}
+                  </TableCell>
+                  <TableCell className={classes.cell}>
+                    {trial["job_id"]}
+                  </TableCell>
+                  <TableCell className={classes.cell}>
+                    {trial["start_time"]}
+                  </TableCell>
+                  {paramNames.map(value => (
+                    <TableCell className={classes.cell} key={value}>
+                      {trial["params"][value]}
+                    </TableCell>
+                  ))}
+                  <TableCell className={classes.cell}>
+                    {trial["status"]}
+                  </TableCell>
+                  {trial["metrics"] &&
+                    metricNames.map(value => (
+                      <TableCell className={classes.cell} key={value}>
+                        {trial["metrics"][value]}
+                      </TableCell>
+                    ))}
+                </TableRow>
+              ))}
+          </TableBody>
+        </Table>
+      </div>
+    );
+  }
+}
+
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(withStyles(styles)(TuneTable));

python/ray/dashboard/client/src/pages/dashboard/tune/TuneTensorBoard.tsx

@@ -0,0 +1,64 @@
+import { Theme } from "@material-ui/core/styles/createMuiTheme";
+import createStyles from "@material-ui/core/styles/createStyles";
+import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles";
+import { connect } from "react-redux";
+import { StoreState } from "../../../store";
+import { dashboardActions } from "../state";
+import React from "react";
+import Typography from "@material-ui/core/Typography";
+
+const styles = (theme: Theme) =>
+  createStyles({
+    root: {
+      backgroundColor: theme.palette.background.paper,
+      padding: theme.spacing(2),
+      "& > :not(:first-child)": {
+        marginTop: theme.spacing(4)
+      }
+    },
+    board: {
+      width: "100%",
+      height: "1000px",
+      border: "none"
+    },
+    warning: {
+      fontSize: "0.8125rem"
+    }
+  });
+
+const mapStateToProps = (state: StoreState) => ({
+  error: state.dashboard.error
+});
+
+const mapDispatchToProps = dashboardActions;
+
+class TuneTensorBoard extends React.Component<
+  WithStyles<typeof styles> &
+    ReturnType<typeof mapStateToProps> &
+    typeof mapDispatchToProps
+> {
+  render() {
+    const { classes, error } = this.props;
+
+    return (
+      <div className={classes.root}>
+        {error === "TypeError: Failed to fetch" && (
+          <Typography className={classes.warning} color="textSecondary">
+            Warning: Tensorboard server closed. View Tensorboard by running
+            "tensorboard --logdir" if not displaying below.
+          </Typography>
+        )}
+        <iframe
+          src="http://localhost:6006/"
+          className={classes.board}
+          title="TensorBoard"
+        ></iframe>
+      </div>
+    );
+  }
+}
+
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(withStyles(styles)(TuneTensorBoard));

python/ray/dashboard/dashboard.py

@@ -38,6 +38,7 @@ import ray.ray_constants as ray_constants
 try:
     from ray.tune.result import DEFAULT_RESULTS_DIR
     from ray.tune import Analysis
+    from tensorboard import program
 except ImportError:
     Analysis = None
 
@@ -754,6 +755,7 @@ class TuneCollector(threading.Thread):
         self._data_lock = threading.Lock()
         self._reload_interval = reload_interval
         self._available = False
+        self._tensor_board_started = False
 
     def get_stats(self):
         with self._data_lock:
@@ -788,6 +790,13 @@ class TuneCollector(threading.Thread):
             if len(df) == 0:
                 continue
 
+            # start TensorBoard server if not started yet
+            if not self._tensor_board_started:
+                tb = program.TensorBoard()
+                tb.configure(argv=[None, "--logdir", self._logdir])
+                tb.launch()
+                self._tensor_board_started = True
+
             self._available = True
 
             # make sure that data will convert to JSON without error
@@ -830,8 +839,10 @@ class TuneCollector(threading.Thread):
 
         # clean data into a form that front-end client can handle
         for trial, details in trial_details.items():
-            details["start_time"] = str(
-                round(os.path.getctime(details["logdir"]), 3))
+            ts = os.path.getctime(details["logdir"])
+            formatted_time = datetime.datetime.fromtimestamp(ts).strftime(
+                "%Y-%m-%d %H:%M:%S")
+            details["start_time"] = formatted_time
             details["params"] = {}
             details["metrics"] = {}
 
@@ -842,12 +853,12 @@ class TuneCollector(threading.Thread):
             # group together config attributes
             for key in config_keys:
                 new_name = key[7:]
-                details["params"][new_name] = str(details[key])
+                details["params"][new_name] = details[key]
                 details.pop(key)
 
             # group together metric attributes
             for key in metric_keys:
-                details["metrics"][key] = str(details[key])
+                details["metrics"][key] = details[key]
                 details.pop(key)
 
             if details["done"]: