From 8adc84ccb91d4888b3995575fd2c96c1c64e2b2a Mon Sep 17 00:00:00 2001 From: aannadi <31830056+aannadi@users.noreply.github.com> Date: Mon, 23 Mar 2020 12:30:51 -0700 Subject: [PATCH] [Dashboard] Add sorted columns and TensorBoard to Tune tab (#7140) --- python/ray/dashboard/client/src/api.ts | 4 +- .../client/src/pages/dashboard/Dashboard.tsx | 2 +- .../client/src/pages/dashboard/Tune.tsx | 162 ------------ .../client/src/pages/dashboard/state.ts | 9 +- .../client/src/pages/dashboard/tune/Tune.tsx | 119 +++++++++ .../src/pages/dashboard/tune/TuneTable.tsx | 241 ++++++++++++++++++ .../pages/dashboard/tune/TuneTensorBoard.tsx | 64 +++++ python/ray/dashboard/dashboard.py | 19 +- 8 files changed, 444 insertions(+), 176 deletions(-) delete mode 100644 python/ray/dashboard/client/src/pages/dashboard/Tune.tsx create mode 100644 python/ray/dashboard/client/src/pages/dashboard/tune/Tune.tsx create mode 100644 python/ray/dashboard/client/src/pages/dashboard/tune/TuneTable.tsx create mode 100644 python/ray/dashboard/client/src/pages/dashboard/tune/TuneTensorBoard.tsx diff --git a/python/ray/dashboard/client/src/api.ts b/python/ray/dashboard/client/src/api.ts index 3de81010c..f52123a87 100644 --- a/python/ray/dashboard/client/src/api.ts +++ b/python/ray/dashboard/client/src/api.ts @@ -219,8 +219,8 @@ export interface TuneTrial { status: string; trial_id: string; job_id: string; - params: { [key: string]: string }; - metrics: { [key: string]: string }; + params: { [key: string]: string | number }; + metrics: { [key: string]: string | number }; } export interface TuneJobResponse { diff --git a/python/ray/dashboard/client/src/pages/dashboard/Dashboard.tsx b/python/ray/dashboard/client/src/pages/dashboard/Dashboard.tsx index 3873b6511..58ae26819 100644 --- a/python/ray/dashboard/client/src/pages/dashboard/Dashboard.tsx +++ b/python/ray/dashboard/client/src/pages/dashboard/Dashboard.tsx @@ -13,7 +13,7 @@ import LogicalView from "./logical-view/LogicalView"; import NodeInfo from "./node-info/NodeInfo"; import RayConfig from "./ray-config/RayConfig"; import { dashboardActions } from "./state"; -import Tune from "./Tune"; +import Tune from "./tune/Tune"; const styles = (theme: Theme) => createStyles({ diff --git a/python/ray/dashboard/client/src/pages/dashboard/Tune.tsx b/python/ray/dashboard/client/src/pages/dashboard/Tune.tsx deleted file mode 100644 index 67292b342..000000000 --- a/python/ray/dashboard/client/src/pages/dashboard/Tune.tsx +++ /dev/null @@ -1,162 +0,0 @@ -import { Theme } from "@material-ui/core/styles/createMuiTheme"; -import createStyles from "@material-ui/core/styles/createStyles"; -import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles"; -import React from "react"; -import { connect } from "react-redux"; -import { getTuneInfo } from "../../api"; -import { StoreState } from "../../store"; -import { dashboardActions } from "./state"; -import Typography from "@material-ui/core/Typography"; -import WarningRoundedIcon from "@material-ui/icons/WarningRounded"; -import Table from "@material-ui/core/Table"; -import TableBody from "@material-ui/core/TableBody"; -import TableCell from "@material-ui/core/TableCell"; -import TableHead from "@material-ui/core/TableHead"; -import TableRow from "@material-ui/core/TableRow"; - -const styles = (theme: Theme) => - createStyles({ - root: { - padding: theme.spacing(2), - "& > :not(:first-child)": { - marginTop: theme.spacing(2) - } - }, - table: { - marginTop: theme.spacing(1) - }, - cell: { - padding: theme.spacing(1), - textAlign: "center", - "&:last-child": { - paddingRight: theme.spacing(1) - } - }, - warning: { - fontSize: "0.8125rem", - marginBottom: theme.spacing(2) - }, - warningIcon: { - fontSize: "1.25em", - verticalAlign: "text-bottom" - } - }); - -const mapStateToProps = (state: StoreState) => ({ - tuneInfo: state.dashboard.tuneInfo -}); - -const mapDispatchToProps = dashboardActions; - -class Tune extends React.Component< - WithStyles & - ReturnType & - typeof mapDispatchToProps -> { - timeout: number = 0; - - refreshTuneInfo = async () => { - try { - const [tuneInfo] = await Promise.all([getTuneInfo()]); - this.props.setTuneInfo({ tuneInfo }); - } catch (error) { - this.props.setError(error.toString()); - } finally { - this.timeout = window.setTimeout(this.refreshTuneInfo, 1000); - } - }; - - async componentDidMount() { - await this.refreshTuneInfo(); - } - - async componentWillUnmount() { - window.clearTimeout(this.timeout); - } - - render() { - const { classes, tuneInfo } = this.props; - - if ( - tuneInfo === null || - Object.keys(tuneInfo["trial_records"]).length === 0 - ) { - return null; - } - - const firstTrial = Object.keys(tuneInfo["trial_records"])[0]; - let paramNames: string[] = []; - if (tuneInfo !== null) { - const paramsDict = tuneInfo["trial_records"][firstTrial]["params"]; - paramNames = Object.keys(paramsDict).filter(k => k !== "args"); - } - - const metricNames = Object.keys( - tuneInfo["trial_records"][firstTrial]["metrics"] - ); - - return ( -
- - Note: This tab - is experimental. - - - - - Trial ID - Job ID - Start Time - {paramNames.map((value, index) => ( - - {value} - - ))} - Status - {metricNames.map((value, index) => ( - - {value} - - ))} - - - - {tuneInfo != null && - Object.keys(tuneInfo["trial_records"]).map(key => ( - - - {tuneInfo["trial_records"][key]["trial_id"]} - - - {tuneInfo["trial_records"][key]["job_id"]} - - - {tuneInfo["trial_records"][key]["start_time"]} - - {paramNames.map((value, index) => ( - - {tuneInfo["trial_records"][key]["params"][value]} - - ))} - - {tuneInfo["trial_records"][key]["status"]} - - {tuneInfo["trial_records"][key]["metrics"] && - metricNames.map((value, index) => ( - - {tuneInfo["trial_records"][key]["metrics"][value]} - - ))} - - ))} - -
-
- ); - } -} - -export default connect( - mapStateToProps, - mapDispatchToProps -)(withStyles(styles)(Tune)); diff --git a/python/ray/dashboard/client/src/pages/dashboard/state.ts b/python/ray/dashboard/client/src/pages/dashboard/state.ts index ea5a6e9c3..68cad9edd 100644 --- a/python/ray/dashboard/client/src/pages/dashboard/state.ts +++ b/python/ray/dashboard/client/src/pages/dashboard/state.ts @@ -52,13 +52,8 @@ const slice = createSlice({ state.rayletInfo = action.payload.rayletInfo; state.lastUpdatedAt = Date.now(); }, - setTuneInfo: ( - state, - action: PayloadAction<{ - tuneInfo: TuneJobResponse; - }> - ) => { - state.tuneInfo = action.payload.tuneInfo; + setTuneInfo: (state, action: PayloadAction) => { + state.tuneInfo = action.payload; state.lastUpdatedAt = Date.now(); }, setTuneAvailability: ( diff --git a/python/ray/dashboard/client/src/pages/dashboard/tune/Tune.tsx b/python/ray/dashboard/client/src/pages/dashboard/tune/Tune.tsx new file mode 100644 index 000000000..0b6dbc3cd --- /dev/null +++ b/python/ray/dashboard/client/src/pages/dashboard/tune/Tune.tsx @@ -0,0 +1,119 @@ +import { Theme } from "@material-ui/core/styles/createMuiTheme"; +import createStyles from "@material-ui/core/styles/createStyles"; +import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles"; +import React from "react"; +import { connect } from "react-redux"; +import { getTuneInfo } from "../../../api"; +import { StoreState } from "../../../store"; +import { dashboardActions } from "../state"; +import Typography from "@material-ui/core/Typography"; +import WarningRoundedIcon from "@material-ui/icons/WarningRounded"; +import Tab from "@material-ui/core/Tab"; +import Tabs from "@material-ui/core/Tabs"; +import TuneTable from "./TuneTable"; +import TuneTensorBoard from "./TuneTensorBoard"; + +const styles = (theme: Theme) => + createStyles({ + root: { + backgroundColor: theme.palette.background.paper + }, + tabs: { + borderBottomColor: theme.palette.divider, + borderBottomStyle: "solid", + borderBottomWidth: 1 + }, + warning: { + fontSize: "0.8125rem" + }, + warningIcon: { + fontSize: "1.25em", + verticalAlign: "text-bottom" + } + }); + +const mapStateToProps = (state: StoreState) => ({ + tuneInfo: state.dashboard.tuneInfo +}); + +const mapDispatchToProps = dashboardActions; + +interface State { + tabIndex: number; +} + +class Tune extends React.Component< + WithStyles & + ReturnType & + typeof mapDispatchToProps, + State +> { + timeout: number = 0; + + state: State = { + tabIndex: 0 + }; + + refreshTuneInfo = async () => { + try { + const tuneInfo = await getTuneInfo(); + this.props.setTuneInfo(tuneInfo); + } catch (error) { + this.props.setError(error.toString()); + } finally { + this.timeout = window.setTimeout(this.refreshTuneInfo, 1000); + } + }; + + async componentDidMount() { + await this.refreshTuneInfo(); + } + + async componentWillUnmount() { + window.clearTimeout(this.timeout); + } + + handleTabChange = (event: React.ChangeEvent<{}>, value: number) => { + this.setState({ + tabIndex: value + }); + }; + + render() { + const { classes } = this.props; + + const { tabIndex } = this.state; + + const tabs = [ + { label: "Table", component: TuneTable }, + { label: "TensorBoard", component: TuneTensorBoard } + ]; + + const SelectedComponent = tabs[tabIndex].component; + return ( +
+ + Note: This tab + is experimental. + + + {tabs.map(({ label }) => ( + + ))} + + +
+ ); + } +} + +export default connect( + mapStateToProps, + mapDispatchToProps +)(withStyles(styles)(Tune)); diff --git a/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTable.tsx b/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTable.tsx new file mode 100644 index 000000000..d3c7225df --- /dev/null +++ b/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTable.tsx @@ -0,0 +1,241 @@ +import { Theme } from "@material-ui/core/styles/createMuiTheme"; +import createStyles from "@material-ui/core/styles/createStyles"; +import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles"; +import React from "react"; +import { connect } from "react-redux"; +import { StoreState } from "../../../store"; +import { dashboardActions } from "../state"; +import Table from "@material-ui/core/Table"; +import TableBody from "@material-ui/core/TableBody"; +import TableCell from "@material-ui/core/TableCell"; +import TableHead from "@material-ui/core/TableHead"; +import TableRow from "@material-ui/core/TableRow"; +import TableSortLabel from "@material-ui/core/TableSortLabel"; +import { TuneTrial } from "../../../api"; + +const styles = (theme: Theme) => + createStyles({ + root: { + padding: theme.spacing(2), + "& > :not(:first-child)": { + marginTop: theme.spacing(2) + } + }, + table: { + marginTop: theme.spacing(1) + }, + cell: { + padding: theme.spacing(1), + textAlign: "right", + "&:last-child": { + paddingRight: theme.spacing(1) + } + } + }); + +const mapStateToProps = (state: StoreState) => ({ + tuneInfo: state.dashboard.tuneInfo +}); + +interface State { + metricParamColumn: string; + ascending: boolean; + sortedColumn: keyof TuneTrial | undefined; +} + +const mapDispatchToProps = dashboardActions; + +class TuneTable extends React.Component< + WithStyles & + ReturnType & + typeof mapDispatchToProps, + State +> { + timeout: number = 0; + + state: State = { + sortedColumn: undefined, + ascending: true, + metricParamColumn: "" + }; + + onColumnClick = (column: keyof TuneTrial, metricParamColumn?: string) => { + let ascending = this.state.ascending; + if (column === this.state.sortedColumn) { + ascending = !ascending; + } else { + ascending = true; + } + this.setState({ + sortedColumn: column, + ascending: ascending + }); + + if (metricParamColumn) { + this.setState({ + metricParamColumn: metricParamColumn + }); + } + }; + + /** + * Replaces all underscores with spaces and capitalizes all words + * in str + */ + humanize = (str: string) => { + var i, + frags = str.split("_"); + for (i = 0; i < frags.length; i++) { + frags[i] = frags[i].charAt(0).toUpperCase() + frags[i].slice(1); + } + return frags.join(" "); + }; + + sortedCell = (name: keyof TuneTrial, chosenMetricParam?: string) => { + const { tuneInfo, classes } = this.props; + const { sortedColumn, ascending, metricParamColumn } = this.state; + let label: "desc" | "asc" = "asc"; + + if (name === sortedColumn && !ascending) { + label = "desc"; + } + + if (tuneInfo === null) { + return; + } + + let onClick = () => this.onColumnClick(name); + if (chosenMetricParam) { + onClick = () => this.onColumnClick(name, chosenMetricParam); + } + + let active = false; + let key: string = name; + if (chosenMetricParam) { + key = chosenMetricParam; + active = chosenMetricParam === metricParamColumn && sortedColumn === name; + } else { + active = name === sortedColumn; + } + + return ( + + + {chosenMetricParam + ? this.humanize(chosenMetricParam) + : this.humanize(name)} + + ); + }; + + sortedTrialRecords = () => { + const { tuneInfo } = this.props; + const { sortedColumn, ascending, metricParamColumn } = this.state; + + if ( + tuneInfo === null || + Object.keys(tuneInfo["trial_records"]).length === 0 + ) { + return null; + } + + const trialDetails = Object.values(tuneInfo["trial_records"]); + + if (!sortedColumn) { + return trialDetails; + } + + let getAttribute = (trial: TuneTrial) => trial[sortedColumn!]; + if (sortedColumn === "metrics" || sortedColumn === "params") { + getAttribute = (trial: TuneTrial) => + trial[sortedColumn!][metricParamColumn]; + } + + if (sortedColumn) { + if (ascending) { + trialDetails.sort((a, b) => + getAttribute(a) > getAttribute(b) ? 1 : -1 + ); + } else if (!ascending) { + trialDetails.sort((a, b) => + getAttribute(a) < getAttribute(b) ? 1 : -1 + ); + } + } + + return trialDetails; + }; + + render() { + const { classes, tuneInfo } = this.props; + + if ( + tuneInfo === null || + Object.keys(tuneInfo["trial_records"]).length === 0 + ) { + return null; + } + + const firstTrial = Object.keys(tuneInfo["trial_records"])[0]; + const paramsDict = tuneInfo["trial_records"][firstTrial]["params"]; + const paramNames = Object.keys(paramsDict).filter(k => k !== "args"); + + const metricNames = Object.keys( + tuneInfo["trial_records"][firstTrial]["metrics"] + ); + + const trialDetails = this.sortedTrialRecords(); + + return ( +
+ + + + {this.sortedCell("trial_id")} + {this.sortedCell("job_id")} + {this.sortedCell("start_time")} + {paramNames.map(value => this.sortedCell("params", value))} + {this.sortedCell("status")} + {metricNames.map(value => this.sortedCell("metrics", value))} + + + + {trialDetails !== null && + trialDetails.map((trial, index) => ( + + + {trial["trial_id"]} + + + {trial["job_id"]} + + + {trial["start_time"]} + + {paramNames.map(value => ( + + {trial["params"][value]} + + ))} + + {trial["status"]} + + {trial["metrics"] && + metricNames.map(value => ( + + {trial["metrics"][value]} + + ))} + + ))} + +
+
+ ); + } +} + +export default connect( + mapStateToProps, + mapDispatchToProps +)(withStyles(styles)(TuneTable)); diff --git a/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTensorBoard.tsx b/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTensorBoard.tsx new file mode 100644 index 000000000..83d43f67c --- /dev/null +++ b/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTensorBoard.tsx @@ -0,0 +1,64 @@ +import { Theme } from "@material-ui/core/styles/createMuiTheme"; +import createStyles from "@material-ui/core/styles/createStyles"; +import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles"; +import { connect } from "react-redux"; +import { StoreState } from "../../../store"; +import { dashboardActions } from "../state"; +import React from "react"; +import Typography from "@material-ui/core/Typography"; + +const styles = (theme: Theme) => + createStyles({ + root: { + backgroundColor: theme.palette.background.paper, + padding: theme.spacing(2), + "& > :not(:first-child)": { + marginTop: theme.spacing(4) + } + }, + board: { + width: "100%", + height: "1000px", + border: "none" + }, + warning: { + fontSize: "0.8125rem" + } + }); + +const mapStateToProps = (state: StoreState) => ({ + error: state.dashboard.error +}); + +const mapDispatchToProps = dashboardActions; + +class TuneTensorBoard extends React.Component< + WithStyles & + ReturnType & + typeof mapDispatchToProps +> { + render() { + const { classes, error } = this.props; + + return ( +
+ {error === "TypeError: Failed to fetch" && ( + + Warning: Tensorboard server closed. View Tensorboard by running + "tensorboard --logdir" if not displaying below. + + )} + +
+ ); + } +} + +export default connect( + mapStateToProps, + mapDispatchToProps +)(withStyles(styles)(TuneTensorBoard)); diff --git a/python/ray/dashboard/dashboard.py b/python/ray/dashboard/dashboard.py index 42c994c88..a4c65f97f 100644 --- a/python/ray/dashboard/dashboard.py +++ b/python/ray/dashboard/dashboard.py @@ -38,6 +38,7 @@ import ray.ray_constants as ray_constants try: from ray.tune.result import DEFAULT_RESULTS_DIR from ray.tune import Analysis + from tensorboard import program except ImportError: Analysis = None @@ -754,6 +755,7 @@ class TuneCollector(threading.Thread): self._data_lock = threading.Lock() self._reload_interval = reload_interval self._available = False + self._tensor_board_started = False def get_stats(self): with self._data_lock: @@ -788,6 +790,13 @@ class TuneCollector(threading.Thread): if len(df) == 0: continue + # start TensorBoard server if not started yet + if not self._tensor_board_started: + tb = program.TensorBoard() + tb.configure(argv=[None, "--logdir", self._logdir]) + tb.launch() + self._tensor_board_started = True + self._available = True # make sure that data will convert to JSON without error @@ -830,8 +839,10 @@ class TuneCollector(threading.Thread): # clean data into a form that front-end client can handle for trial, details in trial_details.items(): - details["start_time"] = str( - round(os.path.getctime(details["logdir"]), 3)) + ts = os.path.getctime(details["logdir"]) + formatted_time = datetime.datetime.fromtimestamp(ts).strftime( + "%Y-%m-%d %H:%M:%S") + details["start_time"] = formatted_time details["params"] = {} details["metrics"] = {} @@ -842,12 +853,12 @@ class TuneCollector(threading.Thread): # group together config attributes for key in config_keys: new_name = key[7:] - details["params"][new_name] = str(details[key]) + details["params"][new_name] = details[key] details.pop(key) # group together metric attributes for key in metric_keys: - details["metrics"][key] = str(details[key]) + details["metrics"][key] = details[key] details.pop(key) if details["done"]: