From 9e31ee991a76548f3910129d28735e09d197d15b Mon Sep 17 00:00:00 2001 From: aannadi <31830056+aannadi@users.noreply.github.com> Date: Fri, 10 Apr 2020 13:27:52 -0700 Subject: [PATCH] =?UTF-8?q?[Dashboard]=20Configure=20Subset=20of=20Paramet?= =?UTF-8?q?ers/Metrics=20and=20show=20Err=E2=80=A6=20(#7726)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors --- python/ray/dashboard/client/src/api.ts | 10 +- .../client/src/pages/dashboard/tune/Tune.tsx | 7 +- .../src/pages/dashboard/tune/TuneErrors.tsx | 133 ++++++++ .../src/pages/dashboard/tune/TuneTable.tsx | 294 +++++++++++++++--- python/ray/dashboard/dashboard.py | 53 +++- 5 files changed, 445 insertions(+), 52 deletions(-) create mode 100644 python/ray/dashboard/client/src/pages/dashboard/tune/TuneErrors.tsx diff --git a/python/ray/dashboard/client/src/api.ts b/python/ray/dashboard/client/src/api.ts index 05992a2c2..32dae8a03 100644 --- a/python/ray/dashboard/client/src/api.ts +++ b/python/ray/dashboard/client/src/api.ts @@ -218,14 +218,22 @@ export type TuneTrial = { training_iteration: number; start_time: string; status: string; - trial_id: string; + trial_id: string | number; job_id: string; params: { [key: string]: string | number }; metrics: { [key: string]: string | number }; + error: string; +}; + +export type TuneError = { + text: string; + job_id: string; + trial_id: string; }; export type TuneJobResponse = { trial_records: { [key: string]: TuneTrial }; + errors: { [key: string]: TuneError }; }; export const getTuneInfo = () => get("/api/tune_info", {}); diff --git a/python/ray/dashboard/client/src/pages/dashboard/tune/Tune.tsx b/python/ray/dashboard/client/src/pages/dashboard/tune/Tune.tsx index aaf2c5429..a2d20b079 100644 --- a/python/ray/dashboard/client/src/pages/dashboard/tune/Tune.tsx +++ b/python/ray/dashboard/client/src/pages/dashboard/tune/Tune.tsx @@ -13,6 +13,7 @@ import { connect } from "react-redux"; import { getTuneInfo } from "../../../api"; import { StoreState } from "../../../store"; import { dashboardActions } from "../state"; +import TuneErrors from "./TuneErrors"; import TuneTable from "./TuneTable"; import TuneTensorBoard from "./TuneTensorBoard"; @@ -83,7 +84,7 @@ class Tune extends React.Component< }; render() { - const { classes } = this.props; + const { classes, tuneInfo } = this.props; const { tabIndex } = this.state; @@ -92,6 +93,10 @@ class Tune extends React.Component< { label: "TensorBoard", component: TuneTensorBoard }, ]; + if (tuneInfo !== null && Object.keys(tuneInfo["errors"]).length > 0) { + tabs.push({ label: "Errors", component: TuneErrors }); + } + const SelectedComponent = tabs[tabIndex].component; return (
diff --git a/python/ray/dashboard/client/src/pages/dashboard/tune/TuneErrors.tsx b/python/ray/dashboard/client/src/pages/dashboard/tune/TuneErrors.tsx new file mode 100644 index 000000000..f068a8ffe --- /dev/null +++ b/python/ray/dashboard/client/src/pages/dashboard/tune/TuneErrors.tsx @@ -0,0 +1,133 @@ +import { + createStyles, + Link, + Table, + TableBody, + TableCell, + TableHead, + TableRow, + Theme, + withStyles, + WithStyles, +} from "@material-ui/core"; +import React from "react"; +import { connect } from "react-redux"; +import DialogWithTitle from "../../../common/DialogWithTitle"; +import NumberedLines from "../../../common/NumberedLines"; +import { StoreState } from "../../../store"; +import { dashboardActions } from "../state"; + +const styles = (theme: Theme) => + createStyles({ + table: { + marginTop: theme.spacing(1), + }, + cell: { + padding: theme.spacing(1), + textAlign: "center", + "&:last-child": { + paddingRight: theme.spacing(1), + }, + }, + }); + +const mapStateToProps = (state: StoreState) => ({ + tuneInfo: state.dashboard.tuneInfo, +}); + +const mapDispatchToProps = dashboardActions; + +type State = { + currentError: string; + open: boolean; +}; + +class TuneErrors extends React.Component< + WithStyles & + ReturnType & + typeof mapDispatchToProps, + State +> { + state: State = { + currentError: "", + open: false, + }; + + handleOpen = (key: string) => { + this.setState({ + open: true, + currentError: key, + }); + }; + + handleClose = () => { + this.setState({ + open: false, + }); + }; + + render() { + const { classes, tuneInfo } = this.props; + const { currentError, open } = this.state; + + if (tuneInfo === null || Object.keys(tuneInfo["errors"]).length === 0) { + return null; + } + + return ( + + + + + Job ID + Trial ID + Trial Directory + Error + + + + {tuneInfo["errors"] !== null && + Object.keys(tuneInfo["errors"]).map((key, index) => ( + + + {tuneInfo["errors"][key]["job_id"]} + + + {tuneInfo["errors"][key]["trial_id"]} + + {key} + + { + this.handleOpen(key); + }} + > + Show Error + + + + ))} + +
+ {open && ( + + {open && ( + + )} + + )} + + ); + } +} + +export default connect( + mapStateToProps, + mapDispatchToProps, +)(withStyles(styles)(TuneErrors)); diff --git a/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTable.tsx b/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTable.tsx index 652158745..d98dbfd86 100644 --- a/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTable.tsx +++ b/python/ray/dashboard/client/src/pages/dashboard/tune/TuneTable.tsx @@ -1,5 +1,12 @@ import { + Checkbox, createStyles, + FormControl, + FormControlLabel, + FormGroup, + FormLabel, + Grid, + Link, Table, TableBody, TableCell, @@ -7,12 +14,14 @@ import { TableRow, TableSortLabel, Theme, - withStyles, WithStyles, + withStyles, } from "@material-ui/core"; import React from "react"; import { connect } from "react-redux"; import { TuneTrial } from "../../../api"; +import DialogWithTitle from "../../../common/DialogWithTitle"; +import NumberedLines from "../../../common/NumberedLines"; import { StoreState } from "../../../store"; import { dashboardActions } from "../state"; @@ -26,6 +35,8 @@ const styles = (theme: Theme) => }, table: { marginTop: theme.spacing(1), + height: "700px", + overflowY: "auto", }, cell: { padding: theme.spacing(1), @@ -34,6 +45,14 @@ const styles = (theme: Theme) => paddingRight: theme.spacing(1), }, }, + checkboxRoot: { + height: "500px", + overflowY: "auto", + overflowX: "auto", + }, + paramChecklist: { + marginBottom: theme.spacing(2), + }, }); const mapStateToProps = (state: StoreState) => ({ @@ -44,6 +63,10 @@ type State = { metricParamColumn: string; ascending: boolean; sortedColumn: keyof TuneTrial | undefined; + metricColumns: string[]; + paramColumns: string[]; + errorTrial: string; + open: boolean; }; const mapDispatchToProps = dashboardActions; @@ -60,6 +83,10 @@ class TuneTable extends React.Component< sortedColumn: undefined, ascending: true, metricParamColumn: "", + metricColumns: [], + paramColumns: [], + errorTrial: "", + open: false, }; onColumnClick = (column: keyof TuneTrial, metricParamColumn?: string) => { @@ -91,7 +118,27 @@ class TuneTable extends React.Component< .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) .join(" "); - sortedCell = (name: keyof TuneTrial, chosenMetricParam?: string) => { + handleOpen = (key: string | number) => { + if (typeof key === "number") { + key = key.toString(); + } + this.setState({ + open: true, + errorTrial: key, + }); + }; + + handleClose = () => { + this.setState({ + open: false, + }); + }; + + sortedCell = ( + name: keyof TuneTrial, + chosenMetricParam?: string, + index?: number, + ) => { const { tuneInfo, classes } = this.props; const { sortedColumn, ascending, metricParamColumn } = this.state; let label: "desc" | "asc" = "asc"; @@ -109,10 +156,14 @@ class TuneTable extends React.Component< onClick = () => this.onColumnClick(name, chosenMetricParam); } + if (!index) { + index = 0; + } + let active = false; - let key: string = name; + let key: string = name + index.toString(); if (chosenMetricParam) { - key = chosenMetricParam; + key = chosenMetricParam + index.toString(); active = chosenMetricParam === metricParamColumn && sortedColumn === name; } else { active = name === sortedColumn; @@ -166,9 +217,95 @@ class TuneTable extends React.Component< return trialDetails; }; + handleMetricChoiceChange = (name: string) => ( + event: React.ChangeEvent, + ) => { + let { metricColumns } = this.state; + if (event.target.checked) { + metricColumns.push(name); + this.setState({ + metricColumns: metricColumns, + }); + } else { + metricColumns = metricColumns.filter((value) => value !== name); + this.setState({ + metricColumns: metricColumns, + }); + } + }; + + metricChoices = (metricNames: string[]) => { + const { metricColumns } = this.state; + + return ( + + Select Metrics + + {metricNames.map((value) => ( + + } + label={value} + /> + ))} + + + ); + }; + + handleParamChoiceChange = (name: string) => ( + event: React.ChangeEvent, + ) => { + let { paramColumns } = this.state; + if (event.target.checked) { + paramColumns.push(name); + this.setState({ + paramColumns: paramColumns, + }); + } else { + paramColumns = paramColumns.filter((value) => value !== name); + this.setState({ + paramColumns: paramColumns, + }); + } + }; + + paramChoices = (paramNames: string[]) => { + const { classes } = this.props; + const { paramColumns } = this.state; + return ( + + Select Parameters + + {paramNames.map((value) => ( + + } + label={value} + /> + ))} + + + ); + }; + render() { const { classes, tuneInfo } = this.props; + const { metricColumns, paramColumns, open, errorTrial } = this.state; + if ( tuneInfo === null || Object.keys(tuneInfo["trial_records"]).length === 0 @@ -180,56 +317,125 @@ class TuneTable extends React.Component< const paramsDict = tuneInfo["trial_records"][firstTrial]["params"]; const paramNames = Object.keys(paramsDict).filter((k) => k !== "args"); + let viewableParams = paramNames; + const paramOptions = paramNames.length > 3; + if (paramOptions) { + if (paramColumns.length === 0) { + this.setState({ + paramColumns: paramNames.slice(0, 3), + }); + } + viewableParams = paramColumns; + } + const metricNames = Object.keys( tuneInfo["trial_records"][firstTrial]["metrics"], ); + let viewableMetrics = metricNames; + const metricOptions = metricNames.length > 3; + if (metricOptions) { + if (metricColumns.length === 0) { + this.setState({ + metricColumns: metricNames.slice(0, 3), + }); + } + viewableMetrics = metricColumns; + } + const trialDetails = this.sortedTrialRecords(); return (
- - - - {this.sortedCell("trial_id")} - {this.sortedCell("job_id")} - {this.sortedCell("start_time")} - {paramNames.map((value) => this.sortedCell("params", value))} - {this.sortedCell("status")} - {metricNames.map((value) => this.sortedCell("metrics", value))} - - - - {trialDetails !== null && - trialDetails.map((trial, index) => ( - - - {trial["trial_id"]} + + {(paramOptions || metricOptions) && ( + + {paramOptions && this.paramChoices(paramNames)} + {metricOptions && this.metricChoices(metricNames)} + + )} + +
+ + + {this.sortedCell("trial_id")} + {this.sortedCell("job_id")} + {this.sortedCell("start_time")} + {viewableParams.map((value, index) => + this.sortedCell("params", value, index), + )} + {this.sortedCell("status")} + {viewableMetrics.map((value, index) => + this.sortedCell("metrics", value, index), + )} + + Error - - {trial["job_id"]} - - - {trial["start_time"]} - - {paramNames.map((value) => ( - - {trial["params"][value]} - - ))} - - {trial["status"]} - - {trial["metrics"] && - metricNames.map((value) => ( - - {trial["metrics"][value]} - - ))} - ))} - -
+ + + {trialDetails !== null && + trialDetails.map((trial, index) => ( + + + {trial["trial_id"]} + + + {trial["job_id"]} + + + {trial["start_time"]} + + {viewableParams.map((value, index) => ( + + {trial["params"][value]} + + ))} + + {trial["status"]} + + {trial["metrics"] && + viewableMetrics.map((value, index) => ( + + {trial["metrics"][value]} + + ))} + + {trial["error"] === "No Error" ? ( + "No Error" + ) : ( + { + this.handleOpen(trial["trial_id"]); + }} + > + Show Error + + )} + + + ))} + + + + + {open && ( + + {open && ( + + )} + + )}
); } diff --git a/python/ray/dashboard/dashboard.py b/python/ray/dashboard/dashboard.py index e1e7ea086..01c01163a 100644 --- a/python/ray/dashboard/dashboard.py +++ b/python/ray/dashboard/dashboard.py @@ -962,13 +962,17 @@ class TuneCollector(threading.Thread): self._reload_interval = reload_interval self._available = False self._tensor_board_started = False + self._errors = {} os.makedirs(self._logdir, exist_ok=True) super().__init__() def get_stats(self): with self._data_lock: - return {"trial_records": copy.deepcopy(self._trial_records)} + return { + "trial_records": copy.deepcopy(self._trial_records), + "errors": copy.deepcopy(self._errors) + } def get_availability(self): with self._data_lock: @@ -980,12 +984,39 @@ class TuneCollector(threading.Thread): self.collect() time.sleep(self._reload_interval) + def collect_errors(self, job_name, df): + sub_dirs = os.listdir(os.path.join(self._logdir, job_name)) + trial_names = filter( + lambda d: os.path.isdir(os.path.join(self._logdir, job_name, d)), + sub_dirs) + for trial in trial_names: + error_path = os.path.join(self._logdir, job_name, trial, + "error.txt") + if os.path.isfile(error_path): + self._available = True + with open(error_path) as f: + text = f.read() + self._errors[str(trial)] = { + "text": text, + "job_id": job_name, + "trial_id": "No Trial ID" + } + other_data = df[df["logdir"].str.contains(trial)] + if len(other_data) > 0: + trial_id = other_data["trial_id"].values[0] + self._errors[str(trial)]["trial_id"] = str(trial_id) + if str(trial_id) in self._trial_records.keys(): + self._trial_records[str(trial_id)]["error"] = text + self._trial_records[str(trial_id)][ + "status"] = "ERROR" + def collect(self): """ Collects and cleans data on the running Tune experiment from the Tune logs so that users can see this information in the front-end client """ + sub_dirs = os.listdir(self._logdir) job_names = filter( lambda d: os.path.isdir(os.path.join(self._logdir, d)), sub_dirs) @@ -996,7 +1027,8 @@ class TuneCollector(threading.Thread): for job_name in job_names: analysis = Analysis(str(os.path.join(self._logdir, job_name))) df = analysis.dataframe() - if len(df) == 0: + + if len(df) == 0 or "trial_id" not in df.columns: continue # start TensorBoard server if not started yet @@ -1009,11 +1041,18 @@ class TuneCollector(threading.Thread): self._available = True # make sure that data will convert to JSON without error - df["trial_id"] = df["trial_id"].astype(str) + df["trial_id_key"] = df["trial_id"].astype(str) df = df.fillna(0) + trial_ids = df["trial_id"] + for i, value in df["trial_id"].iteritems(): + if type(value) != str and type(value) != int: + trial_ids[i] = int(value) + + df["trial_id"] = trial_ids + # convert df to python dict - df = df.set_index("trial_id") + df = df.set_index("trial_id_key") trial_data = df.to_dict(orient="index") # clean data and update class attribute @@ -1021,6 +1060,8 @@ class TuneCollector(threading.Thread): trial_data = self.clean_trials(trial_data, job_name) self._trial_records.update(trial_data) + self.collect_errors(job_name, df) + def clean_trials(self, trial_details, job_name): first_trial = trial_details[list(trial_details.keys())[0]] config_keys = [] @@ -1034,7 +1075,7 @@ class TuneCollector(threading.Thread): "experiment_id", "date", "timestamp", "time_total_s", "pid", "hostname", "node_ip", "time_since_restore", "timesteps_since_restore", "iterations_since_restore", - "experiment_tag" + "experiment_tag", "trial_id" ] # filter attributes into floats, metrics, and config variables @@ -1076,8 +1117,8 @@ class TuneCollector(threading.Thread): details["status"] = "RUNNING" details.pop("done") - details["trial_id"] = trial details["job_id"] = job_name + details["error"] = "No Error" return trial_details