From f87cebb46bc55d0ebb42bd38fd112200e8f57086 Mon Sep 17 00:00:00 2001 From: Cheng Huzi Date: Thu, 3 Jun 2021 22:25:00 -0400 Subject: [PATCH] Add multi-objective support --- optuna_dashboard/app.py | 26 +++-- optuna_dashboard/static/apiClient.ts | 9 +- .../static/components/GraphEdf.tsx | 94 ++++++++++++++----- ...tsx => GraphHyperparameterImportances.tsx} | 86 +++++++++++++++-- .../components/GraphParallelCoordinate.tsx | 65 ++++++++++++- .../static/components/StudyDetail.tsx | 62 +++++------- 6 files changed, 260 insertions(+), 82 deletions(-) rename optuna_dashboard/static/components/{HyperparameterImportances.tsx => GraphHyperparameterImportances.tsx} (52%) diff --git a/optuna_dashboard/app.py b/optuna_dashboard/app.py index 4012117a..a368189e 100644 --- a/optuna_dashboard/app.py +++ b/optuna_dashboard/app.py @@ -214,19 +214,29 @@ def create_app(storage: BaseStorage) -> Bottle: @app.get("/api/studies//param_importances") @handle_json_api_exception - def get_param_importances(study_id: int) -> BottleViewReturn: - # TODO(chenghuzi): add support for selecting params and targets via query parameters. + def get_param_importances(study_id: int,) -> BottleViewReturn: + # TODO(chenghuzi): add support for selecting params via query parameters. response.content_type = "application/json" - study_name = storage.get_study_name_from_id(study_id) - study = Study(study_name=study_name, storage=storage) + objective_id = int(request.params.get("objective_id", 0)) + try: + study_name = storage.get_study_name_from_id(study_id) + study = Study(study_name=study_name, storage=storage) + except KeyError: + response.status = 404 # Not found + return {"reason": f"study_id={study_id} is not found"} - trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE] + n_directions = len(study.directions) + if objective_id >= n_directions: + response.status = 400 # Bad request + return {"reason": f"study_id={study_id} has only {n_directions} direction(s)."} + + completed_trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE] evaluator = None params = None - target = None - if len(trials) > 0: + + if len(completed_trials) > 0: importances = optuna.importance.get_param_importances( - study, evaluator=evaluator, params=params, target=target + study, evaluator=evaluator, params=params, target=lambda t: t.values[objective_id] ) else: importances = {} diff --git a/optuna_dashboard/static/apiClient.ts b/optuna_dashboard/static/apiClient.ts index d6dafdb1..b2504fda 100644 --- a/optuna_dashboard/static/apiClient.ts +++ b/optuna_dashboard/static/apiClient.ts @@ -179,12 +179,17 @@ interface ParamImportancesResponse { } export const getParamImportances = ( - studyId: number + studyId: number, + objectiveId: number = 0 ): Promise => { return axiosInstance .get( `/api/studies/${studyId}/param_importances`, - {} + { + params: { + objective_id: objectiveId, + }, + } ) .then((res) => { return res.data diff --git a/optuna_dashboard/static/components/GraphEdf.tsx b/optuna_dashboard/static/components/GraphEdf.tsx index 7a0e9a23..2abbb9b3 100644 --- a/optuna_dashboard/static/components/GraphEdf.tsx +++ b/optuna_dashboard/static/components/GraphEdf.tsx @@ -1,36 +1,94 @@ import * as plotly from "plotly.js-dist" -import React, { FC, useEffect } from "react" +import React, { FC, useEffect, useState } from "react" +import { + Grid, + FormControl, + FormLabel, + MenuItem, + Select, + Typography, +} from "@material-ui/core" +import { createStyles, makeStyles, Theme } from "@material-ui/core/styles" + +const useStyles = makeStyles((theme: Theme) => + createStyles({ + title: { + margin: "1em 0", + }, + formControl: { + marginBottom: theme.spacing(2), + marginRight: theme.spacing(5), + }, + }) +) const plotDomId = "graph-edf" export const Edf: FC<{ - trials: Trial[] -}> = ({ trials = [] }) => { + study: StudyDetail | null +}> = ({ study = null }) => { + const classes = useStyles() + const [objectiveId, setObjectiveId] = useState(0) + + const handleObjectiveChange = ( + event: React.ChangeEvent<{ value: unknown }> + ) => { + setObjectiveId(event.target.value as number) + } + useEffect(() => { - plotEdf(trials) // TODO(chenghuzi): Support multi-objective studies. - }, [trials]) - return
+ if (study != null) { + plotEdf(study, objectiveId) + } + }, [study, objectiveId]) + return ( + + + + + EDF + + {study !== null && study.directions.length !== 1 ? ( + + Objective ID: + + + ) : null} + + + + +
+ + + ) } -const plotEdf = (trials: Trial[]) => { - // Notice that this implementation is only for single study case - // as it's designed for single study details. +const plotEdf = (study: StudyDetail, objectiveId: number) => { if (document.getElementById(plotDomId) === null) { return } - if (trials.length === 0) { + + const trials: Trial[] = study ? study.trials : [] + const completedTrials = trials.filter((t) => t.state === "Complete") + + if (completedTrials.length === 0) { plotly.react(plotDomId, []) return } const target_name = "Objective Value" - const _target = (t: Trial): number => { - return t.values![0] + const target = (t: Trial): number => { + return t.values![objectiveId] } - const target = _target - const layout: Partial = { title: "Empirical Distribution Function Plot", xaxis: { @@ -46,13 +104,6 @@ const plotEdf = (trials: Trial[]) => { }, } - const completedTrials = trials.filter((t) => t.state === "Complete") - - if (completedTrials.length === 0) { - plotly.react(plotDomId, []) - return - } - const values = completedTrials.map((t) => target(t)) const numValues = values.length const minX = Math.min(...values) @@ -75,6 +126,5 @@ const plotEdf = (trials: Trial[]) => { y: yValues, }, ] - plotly.react(plotDomId, plotData, layout) } diff --git a/optuna_dashboard/static/components/HyperparameterImportances.tsx b/optuna_dashboard/static/components/GraphHyperparameterImportances.tsx similarity index 52% rename from optuna_dashboard/static/components/HyperparameterImportances.tsx rename to optuna_dashboard/static/components/GraphHyperparameterImportances.tsx index 99280bf6..2e14c232 100644 --- a/optuna_dashboard/static/components/HyperparameterImportances.tsx +++ b/optuna_dashboard/static/components/GraphHyperparameterImportances.tsx @@ -1,5 +1,27 @@ import * as plotly from "plotly.js-dist" -import React, { FC, useEffect } from "react" +import React, { FC, useEffect, useState } from "react" +import { + Grid, + FormControl, + FormLabel, + MenuItem, + Select, + Typography, +} from "@material-ui/core" +import { createStyles, makeStyles, Theme } from "@material-ui/core/styles" + +const useStyles = makeStyles((theme: Theme) => + createStyles({ + title: { + margin: "1em 0", + }, + formControl: { + marginBottom: theme.spacing(2), + marginRight: theme.spacing(5), + }, + }) +) + import { getParamImportances } from "../apiClient" const plotDomId = "graph-hyperparameter-importances" @@ -25,18 +47,64 @@ const distributionColors = { CategoricalDistribution: plotlyColorsSequentialBlues.slice(-4)[0], } -export const HyperparameterImportances: FC<{ +export const GraphHyperparameterImportances: FC<{ + study: StudyDetail | null studyId: number - numOfTrials: number -}> = ({ studyId, numOfTrials = 0 }) => { +}> = ({ study = null, studyId }) => { + const classes = useStyles() + const [objectiveId, setObjectiveId] = useState(0) + const numOfTrials = study?.trials.length || 0 + + const handleObjectiveChange = ( + event: React.ChangeEvent<{ value: unknown }> + ) => { + setObjectiveId(event.target.value as number) + } + useEffect(() => { - async function fetchAndPlotParamImportances(studyId: number) { - const paramsImportanceData = await getParamImportances(studyId) + async function fetchAndPlotParamImportances( + studyId: number, + objectiveId: number + ) { + const paramsImportanceData = await getParamImportances( + studyId, + objectiveId + ) plotParamImportances(paramsImportanceData) } - fetchAndPlotParamImportances(studyId) - }, [numOfTrials]) - return
+ + if (numOfTrials > 0) { + fetchAndPlotParamImportances(studyId, objectiveId) + } + }, [numOfTrials, objectiveId]) + + return ( + + + + + Hyperparameter Importance + + {study !== null && study.directions.length !== 1 ? ( + + Objective ID: + + + ) : null} + + + + +
+ + + ) } const plotParamImportances = (paramsImportanceData: ParamImportances) => { diff --git a/optuna_dashboard/static/components/GraphParallelCoordinate.tsx b/optuna_dashboard/static/components/GraphParallelCoordinate.tsx index da7a1d85..b8b2006c 100644 --- a/optuna_dashboard/static/components/GraphParallelCoordinate.tsx +++ b/optuna_dashboard/static/components/GraphParallelCoordinate.tsx @@ -1,17 +1,74 @@ import * as plotly from "plotly.js-dist" -import React, { FC, useEffect } from "react" +import React, { FC, useEffect, useState } from "react" +import { + Grid, + FormControl, + FormLabel, + MenuItem, + Select, + Typography, +} from "@material-ui/core" +import { createStyles, makeStyles, Theme } from "@material-ui/core/styles" + +const useStyles = makeStyles((theme: Theme) => + createStyles({ + title: { + margin: "1em 0", + }, + formControl: { + marginBottom: theme.spacing(2), + marginRight: theme.spacing(5), + }, + }) +) const plotDomId = "graph-parallel-coordinate" export const GraphParallelCoordinate: FC<{ study: StudyDetail | null }> = ({ study = null }) => { + const classes = useStyles() + const [objectiveId, setObjectiveId] = useState(0) + + const handleObjectiveChange = ( + event: React.ChangeEvent<{ value: unknown }> + ) => { + setObjectiveId(event.target.value as number) + } + useEffect(() => { if (study !== null) { - plotCoordinate(study, 0) // TODO(c-bata): Support multi-objective studies. + plotCoordinate(study, objectiveId) } - }, [study]) - return
+ }, [study, objectiveId]) + + return ( + + + + + Parallel cooridinate + + {study !== null && study.directions.length !== 1 ? ( + + Objective ID: + + + ) : null} + + + + +
+ + + ) } const plotCoordinate = (study: StudyDetail, objectiveId: number) => { diff --git a/optuna_dashboard/static/components/StudyDetail.tsx b/optuna_dashboard/static/components/StudyDetail.tsx index 7af107eb..4008d6cf 100644 --- a/optuna_dashboard/static/components/StudyDetail.tsx +++ b/optuna_dashboard/static/components/StudyDetail.tsx @@ -20,7 +20,7 @@ import { Home, Cached } from "@material-ui/icons" import { DataGridColumn, DataGrid } from "./DataGrid" import { GraphParallelCoordinate } from "./GraphParallelCoordinate" -import { HyperparameterImportances } from "./HyperparameterImportances" +import { GraphHyperparameterImportances } from "./GraphHyperparameterImportances" import { Edf } from "./GraphEdf" import { GraphIntermediateValues } from "./GraphIntermediateValues" @@ -189,45 +189,33 @@ export const StudyDetail: FC = () => { ) : null} + + + + + {studyDetail !== null && isSingleObjectiveStudy(studyDetail) ? ( - - - - - - - - - - - - - - - - - ) : null} - {studyDetail !== null && isSingleObjectiveStudy(studyDetail) ? ( - - - - - - - - - - - - - - - + + + + + + ) : null} + + + + + + + + + + {studyDetail !== null ? (