[Dashboard] Configure Subset of Parameters/Metrics and show Err… (#7726)

* Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors * fixup! Subset and Errors
2026-07-07 14:24:44 +08:00 · 2020-04-10 13:27:52 -07:00
parent 4aa68b82fa
commit 9e31ee991a
5 changed files with 445 additions and 52 deletions
@@ -218,14 +218,22 @@ export type TuneTrial = {
  training_iteration: number;
  start_time: string;
  status: string;
-  trial_id: string;
+  trial_id: string | number;
  job_id: string;
  params: { [key: string]: string | number };
  metrics: { [key: string]: string | number };
+  error: string;
+};
+
+export type TuneError = {
+  text: string;
+  job_id: string;
+  trial_id: string;
 };

 export type TuneJobResponse = {
  trial_records: { [key: string]: TuneTrial };
+  errors: { [key: string]: TuneError };
 };

 export const getTuneInfo = () => get<TuneJobResponse>("/api/tune_info", {});
@@ -13,6 +13,7 @@ import { connect } from "react-redux";
 import { getTuneInfo } from "../../../api";
 import { StoreState } from "../../../store";
 import { dashboardActions } from "../state";
+import TuneErrors from "./TuneErrors";
 import TuneTable from "./TuneTable";
 import TuneTensorBoard from "./TuneTensorBoard";

@@ -83,7 +84,7 @@ class Tune extends React.Component<
  };

  render() {
-    const { classes } = this.props;
+    const { classes, tuneInfo } = this.props;

    const { tabIndex } = this.state;

@@ -92,6 +93,10 @@ class Tune extends React.Component<
      { label: "TensorBoard", component: TuneTensorBoard },
    ];

+    if (tuneInfo !== null && Object.keys(tuneInfo["errors"]).length > 0) {
+      tabs.push({ label: "Errors", component: TuneErrors });
+    }
+
    const SelectedComponent = tabs[tabIndex].component;
    return (
      <div className={classes.root}>
@@ -0,0 +1,133 @@
+import {
+  createStyles,
+  Link,
+  Table,
+  TableBody,
+  TableCell,
+  TableHead,
+  TableRow,
+  Theme,
+  withStyles,
+  WithStyles,
+} from "@material-ui/core";
+import React from "react";
+import { connect } from "react-redux";
+import DialogWithTitle from "../../../common/DialogWithTitle";
+import NumberedLines from "../../../common/NumberedLines";
+import { StoreState } from "../../../store";
+import { dashboardActions } from "../state";
+
+const styles = (theme: Theme) =>
+  createStyles({
+    table: {
+      marginTop: theme.spacing(1),
+    },
+    cell: {
+      padding: theme.spacing(1),
+      textAlign: "center",
+      "&:last-child": {
+        paddingRight: theme.spacing(1),
+      },
+    },
+  });
+
+const mapStateToProps = (state: StoreState) => ({
+  tuneInfo: state.dashboard.tuneInfo,
+});
+
+const mapDispatchToProps = dashboardActions;
+
+type State = {
+  currentError: string;
+  open: boolean;
+};
+
+class TuneErrors extends React.Component<
+  WithStyles<typeof styles> &
+    ReturnType<typeof mapStateToProps> &
+    typeof mapDispatchToProps,
+  State
+> {
+  state: State = {
+    currentError: "",
+    open: false,
+  };
+
+  handleOpen = (key: string) => {
+    this.setState({
+      open: true,
+      currentError: key,
+    });
+  };
+
+  handleClose = () => {
+    this.setState({
+      open: false,
+    });
+  };
+
+  render() {
+    const { classes, tuneInfo } = this.props;
+    const { currentError, open } = this.state;
+
+    if (tuneInfo === null || Object.keys(tuneInfo["errors"]).length === 0) {
+      return null;
+    }
+
+    return (
+      <React.Fragment>
+        <Table className={classes.table}>
+          <TableHead>
+            <TableRow>
+              <TableCell className={classes.cell}> Job ID</TableCell>
+              <TableCell className={classes.cell}> Trial ID </TableCell>
+              <TableCell className={classes.cell}> Trial Directory </TableCell>
+              <TableCell className={classes.cell}> Error </TableCell>
+            </TableRow>
+          </TableHead>
+          <TableBody>
+            {tuneInfo["errors"] !== null &&
+              Object.keys(tuneInfo["errors"]).map((key, index) => (
+                <TableRow key={index}>
+                  <TableCell className={classes.cell}>
+                    {tuneInfo["errors"][key]["job_id"]}
+                  </TableCell>
+                  <TableCell className={classes.cell}>
+                    {tuneInfo["errors"][key]["trial_id"]}
+                  </TableCell>
+                  <TableCell className={classes.cell}>{key}</TableCell>
+                  <TableCell className={classes.cell}>
+                    <Link
+                      component="button"
+                      variant="body2"
+                      onClick={() => {
+                        this.handleOpen(key);
+                      }}
+                    >
+                      Show Error
+                    </Link>
+                  </TableCell>
+                </TableRow>
+              ))}
+          </TableBody>
+        </Table>
+        {open && (
+          <DialogWithTitle handleClose={this.handleClose} title="Error Log">
+            {open && (
+              <NumberedLines
+                lines={tuneInfo["errors"][currentError]["text"]
+                  .trim()
+                  .split("\n")}
+              />
+            )}
+          </DialogWithTitle>
+        )}
+      </React.Fragment>
+    );
+  }
+}
+
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps,
+)(withStyles(styles)(TuneErrors));
@@ -1,5 +1,12 @@
 import {
+  Checkbox,
  createStyles,
+  FormControl,
+  FormControlLabel,
+  FormGroup,
+  FormLabel,
+  Grid,
+  Link,
  Table,
  TableBody,
  TableCell,
@@ -7,12 +14,14 @@ import {
  TableRow,
  TableSortLabel,
  Theme,
-  withStyles,
  WithStyles,
+  withStyles,
 } from "@material-ui/core";
 import React from "react";
 import { connect } from "react-redux";
 import { TuneTrial } from "../../../api";
+import DialogWithTitle from "../../../common/DialogWithTitle";
+import NumberedLines from "../../../common/NumberedLines";
 import { StoreState } from "../../../store";
 import { dashboardActions } from "../state";

@@ -26,6 +35,8 @@ const styles = (theme: Theme) =>
    },
    table: {
      marginTop: theme.spacing(1),
+      height: "700px",
+      overflowY: "auto",
    },
    cell: {
      padding: theme.spacing(1),
@@ -34,6 +45,14 @@ const styles = (theme: Theme) =>
        paddingRight: theme.spacing(1),
      },
    },
+    checkboxRoot: {
+      height: "500px",
+      overflowY: "auto",
+      overflowX: "auto",
+    },
+    paramChecklist: {
+      marginBottom: theme.spacing(2),
+    },
  });

 const mapStateToProps = (state: StoreState) => ({
@@ -44,6 +63,10 @@ type State = {
  metricParamColumn: string;
  ascending: boolean;
  sortedColumn: keyof TuneTrial | undefined;
+  metricColumns: string[];
+  paramColumns: string[];
+  errorTrial: string;
+  open: boolean;
 };

 const mapDispatchToProps = dashboardActions;
@@ -60,6 +83,10 @@ class TuneTable extends React.Component<
    sortedColumn: undefined,
    ascending: true,
    metricParamColumn: "",
+    metricColumns: [],
+    paramColumns: [],
+    errorTrial: "",
+    open: false,
  };

  onColumnClick = (column: keyof TuneTrial, metricParamColumn?: string) => {
@@ -91,7 +118,27 @@ class TuneTable extends React.Component<
      .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
      .join(" ");

-  sortedCell = (name: keyof TuneTrial, chosenMetricParam?: string) => {
+  handleOpen = (key: string | number) => {
+    if (typeof key === "number") {
+      key = key.toString();
+    }
+    this.setState({
+      open: true,
+      errorTrial: key,
+    });
+  };
+
+  handleClose = () => {
+    this.setState({
+      open: false,
+    });
+  };
+
+  sortedCell = (
+    name: keyof TuneTrial,
+    chosenMetricParam?: string,
+    index?: number,
+  ) => {
    const { tuneInfo, classes } = this.props;
    const { sortedColumn, ascending, metricParamColumn } = this.state;
    let label: "desc" | "asc" = "asc";
@@ -109,10 +156,14 @@ class TuneTable extends React.Component<
      onClick = () => this.onColumnClick(name, chosenMetricParam);
    }

+    if (!index) {
+      index = 0;
+    }
+
    let active = false;
-    let key: string = name;
+    let key: string = name + index.toString();
    if (chosenMetricParam) {
-      key = chosenMetricParam;
+      key = chosenMetricParam + index.toString();
      active = chosenMetricParam === metricParamColumn && sortedColumn === name;
    } else {
      active = name === sortedColumn;
@@ -166,9 +217,95 @@ class TuneTable extends React.Component<
    return trialDetails;
  };

+  handleMetricChoiceChange = (name: string) => (
+    event: React.ChangeEvent<HTMLInputElement>,
+  ) => {
+    let { metricColumns } = this.state;
+    if (event.target.checked) {
+      metricColumns.push(name);
+      this.setState({
+        metricColumns: metricColumns,
+      });
+    } else {
+      metricColumns = metricColumns.filter((value) => value !== name);
+      this.setState({
+        metricColumns: metricColumns,
+      });
+    }
+  };
+
+  metricChoices = (metricNames: string[]) => {
+    const { metricColumns } = this.state;
+
+    return (
+      <FormControl>
+        <FormLabel component="legend">Select Metrics </FormLabel>
+        <FormGroup>
+          {metricNames.map((value) => (
+            <FormControlLabel
+              control={
+                <Checkbox
+                  checked={metricColumns.includes(value)}
+                  onChange={this.handleMetricChoiceChange(value)}
+                  value={value}
+                  color="primary"
+                />
+              }
+              label={value}
+            />
+          ))}
+        </FormGroup>
+      </FormControl>
+    );
+  };
+
+  handleParamChoiceChange = (name: string) => (
+    event: React.ChangeEvent<HTMLInputElement>,
+  ) => {
+    let { paramColumns } = this.state;
+    if (event.target.checked) {
+      paramColumns.push(name);
+      this.setState({
+        paramColumns: paramColumns,
+      });
+    } else {
+      paramColumns = paramColumns.filter((value) => value !== name);
+      this.setState({
+        paramColumns: paramColumns,
+      });
+    }
+  };
+
+  paramChoices = (paramNames: string[]) => {
+    const { classes } = this.props;
+    const { paramColumns } = this.state;
+    return (
+      <FormControl className={classes.paramChecklist}>
+        <FormLabel component="legend">Select Parameters </FormLabel>
+        <FormGroup>
+          {paramNames.map((value) => (
+            <FormControlLabel
+              control={
+                <Checkbox
+                  checked={paramColumns.includes(value)}
+                  onChange={this.handleParamChoiceChange(value)}
+                  value={value}
+                  color="primary"
+                />
+              }
+              label={value}
+            />
+          ))}
+        </FormGroup>
+      </FormControl>
+    );
+  };
+
  render() {
    const { classes, tuneInfo } = this.props;

+    const { metricColumns, paramColumns, open, errorTrial } = this.state;
+
    if (
      tuneInfo === null ||
      Object.keys(tuneInfo["trial_records"]).length === 0
@@ -180,56 +317,125 @@ class TuneTable extends React.Component<
    const paramsDict = tuneInfo["trial_records"][firstTrial]["params"];
    const paramNames = Object.keys(paramsDict).filter((k) => k !== "args");

+    let viewableParams = paramNames;
+    const paramOptions = paramNames.length > 3;
+    if (paramOptions) {
+      if (paramColumns.length === 0) {
+        this.setState({
+          paramColumns: paramNames.slice(0, 3),
+        });
+      }
+      viewableParams = paramColumns;
+    }
+
    const metricNames = Object.keys(
      tuneInfo["trial_records"][firstTrial]["metrics"],
    );

+    let viewableMetrics = metricNames;
+    const metricOptions = metricNames.length > 3;
+    if (metricOptions) {
+      if (metricColumns.length === 0) {
+        this.setState({
+          metricColumns: metricNames.slice(0, 3),
+        });
+      }
+      viewableMetrics = metricColumns;
+    }
+
    const trialDetails = this.sortedTrialRecords();

    return (
      <div className={classes.root}>
-        <Table className={classes.table}>
-          <TableHead>
-            <TableRow>
-              {this.sortedCell("trial_id")}
-              {this.sortedCell("job_id")}
-              {this.sortedCell("start_time")}
-              {paramNames.map((value) => this.sortedCell("params", value))}
-              {this.sortedCell("status")}
-              {metricNames.map((value) => this.sortedCell("metrics", value))}
-            </TableRow>
-          </TableHead>
-          <TableBody>
-            {trialDetails !== null &&
-              trialDetails.map((trial, index) => (
-                <TableRow key={index}>
-                  <TableCell className={classes.cell}>
-                    {trial["trial_id"]}
+        <Grid container spacing={0}>
+          {(paramOptions || metricOptions) && (
+            <Grid item xs={2} className={classes.checkboxRoot}>
+              {paramOptions && this.paramChoices(paramNames)}
+              {metricOptions && this.metricChoices(metricNames)}
+            </Grid>
+          )}
+          <Grid
+            item
+            xs={paramOptions || metricOptions ? 10 : 12}
+            className={classes.table}
+          >
+            <Table stickyHeader>
+              <TableHead>
+                <TableRow>
+                  {this.sortedCell("trial_id")}
+                  {this.sortedCell("job_id")}
+                  {this.sortedCell("start_time")}
+                  {viewableParams.map((value, index) =>
+                    this.sortedCell("params", value, index),
+                  )}
+                  {this.sortedCell("status")}
+                  {viewableMetrics.map((value, index) =>
+                    this.sortedCell("metrics", value, index),
+                  )}
+                  <TableCell className={classes.cell} key="error">
+                    Error
                  </TableCell>
-                  <TableCell className={classes.cell}>
-                    {trial["job_id"]}
-                  </TableCell>
-                  <TableCell className={classes.cell}>
-                    {trial["start_time"]}
-                  </TableCell>
-                  {paramNames.map((value) => (
-                    <TableCell className={classes.cell} key={value}>
-                      {trial["params"][value]}
-                    </TableCell>
-                  ))}
-                  <TableCell className={classes.cell}>
-                    {trial["status"]}
-                  </TableCell>
-                  {trial["metrics"] &&
-                    metricNames.map((value) => (
-                      <TableCell className={classes.cell} key={value}>
-                        {trial["metrics"][value]}
-                      </TableCell>
-                    ))}
                </TableRow>
-              ))}
-          </TableBody>
-        </Table>
+              </TableHead>
+              <TableBody>
+                {trialDetails !== null &&
+                  trialDetails.map((trial, index) => (
+                    <TableRow key={index}>
+                      <TableCell className={classes.cell}>
+                        {trial["trial_id"]}
+                      </TableCell>
+                      <TableCell className={classes.cell}>
+                        {trial["job_id"]}
+                      </TableCell>
+                      <TableCell className={classes.cell}>
+                        {trial["start_time"]}
+                      </TableCell>
+                      {viewableParams.map((value, index) => (
+                        <TableCell className={classes.cell} key={index}>
+                          {trial["params"][value]}
+                        </TableCell>
+                      ))}
+                      <TableCell className={classes.cell}>
+                        {trial["status"]}
+                      </TableCell>
+                      {trial["metrics"] &&
+                        viewableMetrics.map((value, index) => (
+                          <TableCell className={classes.cell} key={index}>
+                            {trial["metrics"][value]}
+                          </TableCell>
+                        ))}
+                      <TableCell className={classes.cell}>
+                        {trial["error"] === "No Error" ? (
+                          "No Error"
+                        ) : (
+                          <Link
+                            component="button"
+                            variant="body2"
+                            onClick={() => {
+                              this.handleOpen(trial["trial_id"]);
+                            }}
+                          >
+                            Show Error
+                          </Link>
+                        )}
+                      </TableCell>
+                    </TableRow>
+                  ))}
+              </TableBody>
+            </Table>
+          </Grid>
+        </Grid>
+        {open && (
+          <DialogWithTitle handleClose={this.handleClose} title="Error Log">
+            {open && (
+              <NumberedLines
+                lines={tuneInfo["trial_records"][errorTrial]["error"]
+                  .trim()
+                  .split("\n")}
+              />
+            )}
+          </DialogWithTitle>
+        )}
      </div>
    );
  }
@@ -962,13 +962,17 @@ class TuneCollector(threading.Thread):
        self._reload_interval = reload_interval
        self._available = False
        self._tensor_board_started = False
+        self._errors = {}

        os.makedirs(self._logdir, exist_ok=True)
        super().__init__()

    def get_stats(self):
        with self._data_lock:
-            return {"trial_records": copy.deepcopy(self._trial_records)}
+            return {
+                "trial_records": copy.deepcopy(self._trial_records),
+                "errors": copy.deepcopy(self._errors)
+            }

    def get_availability(self):
        with self._data_lock:
@@ -980,12 +984,39 @@ class TuneCollector(threading.Thread):
                self.collect()
            time.sleep(self._reload_interval)

+    def collect_errors(self, job_name, df):
+        sub_dirs = os.listdir(os.path.join(self._logdir, job_name))
+        trial_names = filter(
+            lambda d: os.path.isdir(os.path.join(self._logdir, job_name, d)),
+            sub_dirs)
+        for trial in trial_names:
+            error_path = os.path.join(self._logdir, job_name, trial,
+                                      "error.txt")
+            if os.path.isfile(error_path):
+                self._available = True
+                with open(error_path) as f:
+                    text = f.read()
+                    self._errors[str(trial)] = {
+                        "text": text,
+                        "job_id": job_name,
+                        "trial_id": "No Trial ID"
+                    }
+                    other_data = df[df["logdir"].str.contains(trial)]
+                    if len(other_data) > 0:
+                        trial_id = other_data["trial_id"].values[0]
+                        self._errors[str(trial)]["trial_id"] = str(trial_id)
+                        if str(trial_id) in self._trial_records.keys():
+                            self._trial_records[str(trial_id)]["error"] = text
+                            self._trial_records[str(trial_id)][
+                                "status"] = "ERROR"
+
    def collect(self):
        """
        Collects and cleans data on the running Tune experiment from the
        Tune logs so that users can see this information in the front-end
        client
        """
+
        sub_dirs = os.listdir(self._logdir)
        job_names = filter(
            lambda d: os.path.isdir(os.path.join(self._logdir, d)), sub_dirs)
@@ -996,7 +1027,8 @@ class TuneCollector(threading.Thread):
        for job_name in job_names:
            analysis = Analysis(str(os.path.join(self._logdir, job_name)))
            df = analysis.dataframe()
-            if len(df) == 0:
+
+            if len(df) == 0 or "trial_id" not in df.columns:
                continue

            # start TensorBoard server if not started yet
@@ -1009,11 +1041,18 @@ class TuneCollector(threading.Thread):
            self._available = True

            # make sure that data will convert to JSON without error
-            df["trial_id"] = df["trial_id"].astype(str)
+            df["trial_id_key"] = df["trial_id"].astype(str)
            df = df.fillna(0)

+            trial_ids = df["trial_id"]
+            for i, value in df["trial_id"].iteritems():
+                if type(value) != str and type(value) != int:
+                    trial_ids[i] = int(value)
+
+            df["trial_id"] = trial_ids
+
            # convert df to python dict
-            df = df.set_index("trial_id")
+            df = df.set_index("trial_id_key")
            trial_data = df.to_dict(orient="index")

            # clean data and update class attribute
@@ -1021,6 +1060,8 @@ class TuneCollector(threading.Thread):
                trial_data = self.clean_trials(trial_data, job_name)
                self._trial_records.update(trial_data)

+            self.collect_errors(job_name, df)
+
    def clean_trials(self, trial_details, job_name):
        first_trial = trial_details[list(trial_details.keys())[0]]
        config_keys = []
@@ -1034,7 +1075,7 @@ class TuneCollector(threading.Thread):
            "experiment_id", "date", "timestamp", "time_total_s", "pid",
            "hostname", "node_ip", "time_since_restore",
            "timesteps_since_restore", "iterations_since_restore",
-            "experiment_tag"
+            "experiment_tag", "trial_id"
        ]

        # filter attributes into floats, metrics, and config variables
@@ -1076,8 +1117,8 @@ class TuneCollector(threading.Thread):
                details["status"] = "RUNNING"
            details.pop("done")

-            details["trial_id"] = trial
            details["job_id"] = job_name
+            details["error"] = "No Error"

        return trial_details