From 1a7ac3ff3c828c9554d4b410eb2e70017305f47b Mon Sep 17 00:00:00 2001 From: Henry Cui Date: Sat, 27 Feb 2021 16:16:49 +0900 Subject: [PATCH 01/18] Add simple pareto front plot --- optuna_dashboard/static/apiClient.ts | 9 +++ .../static/components/GraphParetoFront.tsx | 60 +++++++++++++++++++ .../static/components/StudyDetail.tsx | 8 +++ optuna_dashboard/static/types/index.d.ts | 1 + 4 files changed, 78 insertions(+) create mode 100644 optuna_dashboard/static/components/GraphParetoFront.tsx diff --git a/optuna_dashboard/static/apiClient.ts b/optuna_dashboard/static/apiClient.ts index cd68c40a..befda07a 100644 --- a/optuna_dashboard/static/apiClient.ts +++ b/optuna_dashboard/static/apiClient.ts @@ -34,11 +34,17 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } +const convertTrialResponseList = (res: TrialResponse[]): Trial[] => { + return res.map((trial): Trial => convertTrialResponse(trial)) +} + + interface StudyDetailResponse { name: string datetime_start: string directions: StudyDirection[] best_trial?: TrialResponse + best_trials?: TrialResponse[] trials: TrialResponse[] } @@ -58,6 +64,9 @@ export const getStudyDetailAPI = (studyId: number): Promise => { best_trial: res.data.best_trial ? convertTrialResponse(res.data.best_trial) : undefined, + best_trials: res.data.best_trials + ? convertTrialResponseList(res.data.best_trials) + : undefined, trials: trials, } }) diff --git a/optuna_dashboard/static/components/GraphParetoFront.tsx b/optuna_dashboard/static/components/GraphParetoFront.tsx new file mode 100644 index 00000000..196e0a61 --- /dev/null +++ b/optuna_dashboard/static/components/GraphParetoFront.tsx @@ -0,0 +1,60 @@ +import * as plotly from "plotly.js-dist" +import React, { FC, useEffect } from "react" + +const plotDomId = "graph-pareto-front" + +export const GraphParetoFront: FC<{ + study: StudyDetail | null +}> = ({ study = null }) => { + + useEffect(() => { + if (study != null) { + plotParetoFront(study) + } + }, [study]) + + return
+} + +const plotParetoFront = (study: StudyDetail) => { + if (document.getElementById(plotDomId) === null) { + return + } + + if (study.directions.length != 2) { + return + } + + const layout: Partial = { + title: "Pareto-front plot", + margin: { + l: 50, + r: 50, + b: 0, + }, + } + + const trials: Trial[] = (study !== null) && study.best_trials ? study.best_trials : [] + console.log(study.best_trials) + console.log('length', trials.length) + if (trials.length === 0) { + plotly.react(plotDomId, [], layout) + return + } + + const pointColors = Array(trials.length).fill("blue") + + const plotData: Partial[] = [ + { + type: "scatter", + x: trials.map((t: Trial): number => t.values![0]), + y: trials.map((t: Trial): number => t.values![1]), + mode: "markers", + marker: { + color: pointColors, + }, + }, + ] + + plotly.react(plotDomId, plotData, layout) +} diff --git a/optuna_dashboard/static/components/StudyDetail.tsx b/optuna_dashboard/static/components/StudyDetail.tsx index d4068fec..85e4dc7e 100644 --- a/optuna_dashboard/static/components/StudyDetail.tsx +++ b/optuna_dashboard/static/components/StudyDetail.tsx @@ -23,6 +23,7 @@ import { GraphParallelCoordinate } from "./GraphParallelCoordinate" import { GraphIntermediateValues } from "./GraphIntermediateValues" import { GraphSlice } from "./GraphSlice" import { GraphHistory } from "./GraphHistory" +import { GraphParetoFront } from "./GraphParetoFront" import { actionCreator } from "../action" import { studyDetailsState } from "../state" @@ -203,6 +204,13 @@ export const StudyDetail: FC = () => { ) : null} + {studyDetail !== null && !isSingleObjectiveStudy(studyDetail) ? ( + + + + + + ) : null} diff --git a/optuna_dashboard/static/types/index.d.ts b/optuna_dashboard/static/types/index.d.ts index a534a02f..295a24c7 100644 --- a/optuna_dashboard/static/types/index.d.ts +++ b/optuna_dashboard/static/types/index.d.ts @@ -54,6 +54,7 @@ declare interface StudyDetail { directions: StudyDirection[] datetime_start: Date best_trial?: Trial + best_trials?: Trial[] trials: Trial[] } From 2a0b7e0d625fca3a242153edc646c6965bf4f260 Mon Sep 17 00:00:00 2001 From: Henry Cui Date: Sat, 27 Feb 2021 17:50:04 +0900 Subject: [PATCH 02/18] Enabled pareto front 2d plot --- optuna_dashboard/static/apiClient.ts | 9 ---- .../static/components/GraphParetoFront.tsx | 46 +++++++++++++++---- optuna_dashboard/static/types/index.d.ts | 1 - 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/optuna_dashboard/static/apiClient.ts b/optuna_dashboard/static/apiClient.ts index befda07a..cd68c40a 100644 --- a/optuna_dashboard/static/apiClient.ts +++ b/optuna_dashboard/static/apiClient.ts @@ -34,17 +34,11 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } -const convertTrialResponseList = (res: TrialResponse[]): Trial[] => { - return res.map((trial): Trial => convertTrialResponse(trial)) -} - - interface StudyDetailResponse { name: string datetime_start: string directions: StudyDirection[] best_trial?: TrialResponse - best_trials?: TrialResponse[] trials: TrialResponse[] } @@ -64,9 +58,6 @@ export const getStudyDetailAPI = (studyId: number): Promise => { best_trial: res.data.best_trial ? convertTrialResponse(res.data.best_trial) : undefined, - best_trials: res.data.best_trials - ? convertTrialResponseList(res.data.best_trials) - : undefined, trials: trials, } }) diff --git a/optuna_dashboard/static/components/GraphParetoFront.tsx b/optuna_dashboard/static/components/GraphParetoFront.tsx index 196e0a61..99bba3f3 100644 --- a/optuna_dashboard/static/components/GraphParetoFront.tsx +++ b/optuna_dashboard/static/components/GraphParetoFront.tsx @@ -21,7 +21,8 @@ const plotParetoFront = (study: StudyDetail) => { return } - if (study.directions.length != 2) { + const dim: number = study.directions.length + if (dim != 2) { return } @@ -34,21 +35,50 @@ const plotParetoFront = (study: StudyDetail) => { }, } - const trials: Trial[] = (study !== null) && study.best_trials ? study.best_trials : [] - console.log(study.best_trials) - console.log('length', trials.length) - if (trials.length === 0) { + const trials: Trial[] = study !== null ? study.trials : [] + const completedTrials = trials.filter( + (t) => t.state === "Complete" + ) + + if (completedTrials.length === 0) { plotly.react(plotDomId, [], layout) return } - const pointColors = Array(trials.length).fill("blue") + const normalizedValues: number[][] = [] + completedTrials.forEach((t) => { + if (t.values && t.values.length == dim) { + let values: number[] = t.values + values.forEach((v: number, i: number) => { + if (study.directions[i] === "maximize") { + values[i] = -v + } + }) + normalizedValues.push(values) + } + }) + + const pointColors: string[] = [] + normalizedValues.forEach((values0: number[], i: number) => { + let dominated: boolean = false + + dominated = normalizedValues.some((values1: number[], j: number) => { + if (i === j) { + return false + } + return values0.every((value0: number, k: number) => { + return value0 <= values1[k] + }) + }) + + if (dominated) { pointColors.push("blue") } else { pointColors.push("red") } + }) const plotData: Partial[] = [ { type: "scatter", - x: trials.map((t: Trial): number => t.values![0]), - y: trials.map((t: Trial): number => t.values![1]), + x: completedTrials.map((t: Trial): number => t.values![0]), + y: completedTrials.map((t: Trial): number => t.values![1]), mode: "markers", marker: { color: pointColors, diff --git a/optuna_dashboard/static/types/index.d.ts b/optuna_dashboard/static/types/index.d.ts index 295a24c7..a534a02f 100644 --- a/optuna_dashboard/static/types/index.d.ts +++ b/optuna_dashboard/static/types/index.d.ts @@ -54,7 +54,6 @@ declare interface StudyDetail { directions: StudyDirection[] datetime_start: Date best_trial?: Trial - best_trials?: Trial[] trials: Trial[] } From c4751307f1eb5bd8aff938d40c5c1489cd3d9fe0 Mon Sep 17 00:00:00 2001 From: Henry Cui Date: Sun, 28 Feb 2021 00:39:22 +0900 Subject: [PATCH 03/18] Debugged and improved --- .../static/components/GraphParetoFront.tsx | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/optuna_dashboard/static/components/GraphParetoFront.tsx b/optuna_dashboard/static/components/GraphParetoFront.tsx index 99bba3f3..a1bbd040 100644 --- a/optuna_dashboard/static/components/GraphParetoFront.tsx +++ b/optuna_dashboard/static/components/GraphParetoFront.tsx @@ -25,7 +25,6 @@ const plotParetoFront = (study: StudyDetail) => { if (dim != 2) { return } - const layout: Partial = { title: "Pareto-front plot", margin: { @@ -45,29 +44,28 @@ const plotParetoFront = (study: StudyDetail) => { return } - const normalizedValues: number[][] = [] + let normalizedValues: number[][] = [] completedTrials.forEach((t) => { if (t.values && t.values.length == dim) { - let values: number[] = t.values - values.forEach((v: number, i: number) => { - if (study.directions[i] === "maximize") { - values[i] = -v + const trialValues = t.values.map( + (v: number, i: number) => { + return (study.directions[i] === "minimize") ? v : -v } - }) - normalizedValues.push(values) + ) + normalizedValues.push(trialValues) } }) const pointColors: string[] = [] normalizedValues.forEach((values0: number[], i: number) => { - let dominated: boolean = false + let dominated = false dominated = normalizedValues.some((values1: number[], j: number) => { if (i === j) { return false } return values0.every((value0: number, k: number) => { - return value0 <= values1[k] + return values1[k] <= value0 }) }) @@ -77,12 +75,22 @@ const plotParetoFront = (study: StudyDetail) => { const plotData: Partial[] = [ { type: "scatter", - x: completedTrials.map((t: Trial): number => t.values![0]), - y: completedTrials.map((t: Trial): number => t.values![1]), + x: completedTrials.map((t: Trial): number => { return t.values![0]}), + y: completedTrials.map((t: Trial): number => { return t.values![1]}), mode: "markers", + xaxis: "Objective 0", + yaxis: "Objective 1", marker: { - color: pointColors, + color: pointColors }, + text: completedTrials.map((t: Trial): string => { + return JSON.stringify({ + "number": t.number, + "values": t.values, + "params": t.params, + }, null, 2).replaceAll("\n", "
") + }), + hovertemplate: "%{text}", }, ] From 2ef70332b0a873ea4cd1f47b02f7b07d7ee2b667 Mon Sep 17 00:00:00 2001 From: Henry Cui Date: Sun, 28 Feb 2021 00:40:54 +0900 Subject: [PATCH 04/18] Change let to const for normalizedValues --- optuna_dashboard/static/components/GraphParetoFront.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/static/components/GraphParetoFront.tsx b/optuna_dashboard/static/components/GraphParetoFront.tsx index a1bbd040..d8bcb3f3 100644 --- a/optuna_dashboard/static/components/GraphParetoFront.tsx +++ b/optuna_dashboard/static/components/GraphParetoFront.tsx @@ -44,7 +44,7 @@ const plotParetoFront = (study: StudyDetail) => { return } - let normalizedValues: number[][] = [] + const normalizedValues: number[][] = [] completedTrials.forEach((t) => { if (t.values && t.values.length == dim) { const trialValues = t.values.map( From 6f12ac71e38c9c069876134b8558c5ef967e0fef Mon Sep 17 00:00:00 2001 From: Henry Cui Date: Sun, 28 Feb 2021 15:55:29 +0900 Subject: [PATCH 05/18] Apply fmt --- .../static/components/GraphParetoFront.tsx | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/optuna_dashboard/static/components/GraphParetoFront.tsx b/optuna_dashboard/static/components/GraphParetoFront.tsx index d8bcb3f3..f2a6f0cc 100644 --- a/optuna_dashboard/static/components/GraphParetoFront.tsx +++ b/optuna_dashboard/static/components/GraphParetoFront.tsx @@ -6,7 +6,6 @@ const plotDomId = "graph-pareto-front" export const GraphParetoFront: FC<{ study: StudyDetail | null }> = ({ study = null }) => { - useEffect(() => { if (study != null) { plotParetoFront(study) @@ -35,9 +34,7 @@ const plotParetoFront = (study: StudyDetail) => { } const trials: Trial[] = study !== null ? study.trials : [] - const completedTrials = trials.filter( - (t) => t.state === "Complete" - ) + const completedTrials = trials.filter((t) => t.state === "Complete") if (completedTrials.length === 0) { plotly.react(plotDomId, [], layout) @@ -47,11 +44,9 @@ const plotParetoFront = (study: StudyDetail) => { const normalizedValues: number[][] = [] completedTrials.forEach((t) => { if (t.values && t.values.length == dim) { - const trialValues = t.values.map( - (v: number, i: number) => { - return (study.directions[i] === "minimize") ? v : -v - } - ) + const trialValues = t.values.map((v: number, i: number) => { + return study.directions[i] === "minimize" ? v : -v + }) normalizedValues.push(trialValues) } }) @@ -69,28 +64,40 @@ const plotParetoFront = (study: StudyDetail) => { }) }) - if (dominated) { pointColors.push("blue") } else { pointColors.push("red") } + if (dominated) { + pointColors.push("blue") + } else { + pointColors.push("red") + } }) const plotData: Partial[] = [ { - type: "scatter", - x: completedTrials.map((t: Trial): number => { return t.values![0]}), - y: completedTrials.map((t: Trial): number => { return t.values![1]}), - mode: "markers", - xaxis: "Objective 0", - yaxis: "Objective 1", - marker: { - color: pointColors - }, - text: completedTrials.map((t: Trial): string => { - return JSON.stringify({ - "number": t.number, - "values": t.values, - "params": t.params, - }, null, 2).replaceAll("\n", "
") - }), - hovertemplate: "%{text}", + type: "scatter", + x: completedTrials.map((t: Trial): number => { + return t.values![0] + }), + y: completedTrials.map((t: Trial): number => { + return t.values![1] + }), + mode: "markers", + xaxis: "Objective 0", + yaxis: "Objective 1", + marker: { + color: pointColors, + }, + text: completedTrials.map((t: Trial): string => { + return JSON.stringify( + { + number: t.number, + values: t.values, + params: t.params, + }, + null, + 2 + ).replaceAll("\n", "
") + }), + hovertemplate: "%{text}", }, ] From 981b30e9df20d1b92e222ef9d6d8b51ffb908bc0 Mon Sep 17 00:00:00 2001 From: Cheng Huzi Date: Wed, 10 Mar 2021 01:06:59 -0500 Subject: [PATCH 06/18] Add hyperparameter importances chart --- optuna_dashboard/app.py | 44 +++++++++- optuna_dashboard/static/apiClient.ts | 20 +++++ .../components/HyperparameterImportances.tsx | 88 +++++++++++++++++++ .../static/components/StudyDetail.tsx | 12 +++ optuna_dashboard/static/types/index.d.ts | 18 ++++ 5 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 optuna_dashboard/static/components/HyperparameterImportances.tsx diff --git a/optuna_dashboard/app.py b/optuna_dashboard/app.py index e1af9ca9..d7891a42 100644 --- a/optuna_dashboard/app.py +++ b/optuna_dashboard/app.py @@ -8,10 +8,11 @@ import traceback from typing import Union, Dict, List, Optional, TypeVar, Callable, Any, cast from bottle import Bottle, BaseResponse, redirect, request, response, static_file +import optuna from optuna.exceptions import DuplicatedStudyError from optuna.storages import BaseStorage -from optuna.trial import FrozenTrial -from optuna.study import StudyDirection, StudySummary +from optuna.trial import FrozenTrial, TrialState +from optuna.study import StudyDirection, StudySummary, Study from . import serializer @@ -94,6 +95,13 @@ def get_trials( return trials +def get_distribution_name(param_name: str, study: Study) -> str: + for trial in study.trials: + if param_name in trial.distributions: + return trial.distributions[param_name].__class__.__name__ + assert False + + def create_app(storage: BaseStorage) -> Bottle: app = Bottle() @@ -185,6 +193,38 @@ def create_app(storage: BaseStorage) -> Bottle: trials = get_trials(storage, study_id) return serializer.serialize_study_detail(summary, trials) + @app.get("/api/studies//param_importances") + @handle_json_api_exception + def get_param_importances(study_id: int) -> BottleViewReturn: + # TODO: add support for selecting params and targets via query parameters. + response.content_type = "application/json" + study_name = storage.get_study_name_from_id(study_id) + study = Study(study_name=study_name, storage=storage) + + trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE] + if len(trials) == 0: + return "" + evaluator = None + params = None + target = None + importances = optuna.importance.get_param_importances( + study, evaluator=evaluator, params=params, target=target + ) + if target is None: + target_name = "Objective Value" + + return { + "target_name": target_name, + "param_importances": [ + { + "name": i[0], + "importance": i[1], + "distribution": get_distribution_name(i[0], study), + } + for i in importances.items() + ], + } + @app.get("/static/") def send_static(filename: str) -> BottleViewReturn: return static_file(filename, root=STATIC_DIR) diff --git a/optuna_dashboard/static/apiClient.ts b/optuna_dashboard/static/apiClient.ts index cf9348f9..0e923dca 100644 --- a/optuna_dashboard/static/apiClient.ts +++ b/optuna_dashboard/static/apiClient.ts @@ -168,3 +168,23 @@ export const deleteStudyAPI = (studyId: number) => { return {} }) } + +interface ParamImportancesResponse { + target_name: string + param_importances: ParamImportance[] +} + +export const getParamImportances = ( + studyId: number +): Promise => { + return axiosInstance + .get( + `/api/studies/${studyId}/param_importances`, + {} + ) + .then((res) => { + return res.data + }) +} + +export default getParamImportances diff --git a/optuna_dashboard/static/components/HyperparameterImportances.tsx b/optuna_dashboard/static/components/HyperparameterImportances.tsx new file mode 100644 index 00000000..59904769 --- /dev/null +++ b/optuna_dashboard/static/components/HyperparameterImportances.tsx @@ -0,0 +1,88 @@ +import * as plotly from "plotly.js-dist" +import React, { FC, useEffect } from "react" +import { getParamImportances } from "../apiClient" +const plotDomId = "graph-hyperparameter-importances" + +// To match colors used by plot_param_importances in optuna. +const plotlyColorsSequentialBlues = [ + "rgb(247,251,255)", + "rgb(222,235,247)", + "rgb(198,219,239)", + "rgb(158,202,225)", + "rgb(107,174,214)", + "rgb(66,146,198)", + "rgb(33,113,181)", + "rgb(8,81,156)", + "rgb(8,48,107)", +] + +const distributionColors = { + UniformDistribution: plotlyColorsSequentialBlues.slice(-1)[0], + LogUniformDistribution: plotlyColorsSequentialBlues.slice(-1)[0], + DiscreteUniformDistribution: plotlyColorsSequentialBlues.slice(-1)[0], + IntUniformDistribution: plotlyColorsSequentialBlues.slice(-2)[0], + IntLogUniformDistribution: plotlyColorsSequentialBlues.slice(-2)[0], + CategoricalDistribution: plotlyColorsSequentialBlues.slice(-4)[0], +} + +export const HyperparameterImportances: FC<{ + studyId: number +}> = ({ studyId }) => { + useEffect(() => { + async function fetchAndPlotParamImportances(studyId: number) { + const paramsImportanceData = await getParamImportances(studyId) + plotParamImportances(paramsImportanceData) + } + fetchAndPlotParamImportances(studyId) + }) + return
+} + +const plotParamImportances = (paramsImportanceData: ParamImportances) => { + if (document.getElementById(plotDomId) === null) { + return + } + const param_importances = paramsImportanceData.param_importances.reverse() + const importance_values = param_importances.map((p) => p.importance) + const param_names = param_importances.map((p) => p.name) + const param_colors = param_importances.map( + (p) => distributionColors[p.distribution] + ) + const param_hover_templates = param_importances.map( + (p) => `${p.name} (${p.distribution}): ${p.importance} ` + ) + console.log("param_colors", param_colors) + + const layout: Partial = { + title: "Hyperparameter Importance", + xaxis: { + title: `Importance for ${paramsImportanceData.target_name}`, + }, + yaxis: { + title: "Hyperparameter", + }, + margin: { + l: 50, + r: 50, + b: 50, + }, + showlegend: false, + } + + const plotData: Partial[] = [ + { + type: "bar", + orientation: "h", + x: importance_values, + y: param_names, + text: importance_values.map((v) => String(v.toFixed(2))), + textposition: "outside", + hovertemplate: param_hover_templates, + marker: { + color: param_colors, + }, + }, + ] + + plotly.react(plotDomId, plotData, layout) +} diff --git a/optuna_dashboard/static/components/StudyDetail.tsx b/optuna_dashboard/static/components/StudyDetail.tsx index bd47d1ae..56e84507 100644 --- a/optuna_dashboard/static/components/StudyDetail.tsx +++ b/optuna_dashboard/static/components/StudyDetail.tsx @@ -20,6 +20,7 @@ import { Home, Cached } from "@material-ui/icons" import { DataGridColumn, DataGrid } from "./DataGrid" import { GraphParallelCoordinate } from "./GraphParallelCoordinate" +import { HyperparameterImportances } from "./HyperparameterImportances" import { GraphIntermediateValues } from "./GraphIntermediateValues" import { GraphSlice } from "./GraphSlice" import { GraphHistory } from "./GraphHistory" @@ -196,6 +197,17 @@ export const StudyDetail: FC = () => { ) : null} + {studyDetail !== null && isSingleObjectiveStudy(studyDetail) ? ( + + + + + + + + + + ) : null} {studyDetail !== null ? ( diff --git a/optuna_dashboard/static/types/index.d.ts b/optuna_dashboard/static/types/index.d.ts index cc320769..2ae3ee30 100644 --- a/optuna_dashboard/static/types/index.d.ts +++ b/optuna_dashboard/static/types/index.d.ts @@ -9,6 +9,13 @@ declare const URL_PREFIX: string type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type StudyDirection = "maximize" | "minimize" | "not_set" +type Distribution = + | "UniformDistribution" + | "LogUniformDistribution" + | "DiscreteUniformDistribution" + | "IntUniformDistribution" + | "IntLogUniformDistribution" + | "CategoricalDistribution" declare interface TrialIntermediateValue { step: number @@ -20,6 +27,12 @@ declare interface TrialParam { value: string } +declare interface ParamImportance { + name: string + importance: number + distribution: Distribution +} + declare interface Attribute { key: string value: string @@ -60,3 +73,8 @@ declare interface StudyDetail { declare interface StudyDetails { [study_id: string]: StudyDetail } + +declare interface ParamImportances { + target_name: string + param_importances: ParamImportance[] +} From d228ba74df2744d22034c2bbf80c7ddea0a4f6cd Mon Sep 17 00:00:00 2001 From: Huzi Cheng Date: Thu, 11 Mar 2021 18:32:18 -0500 Subject: [PATCH 07/18] Update optuna_dashboard/app.py Co-authored-by: Masashi Shibata --- optuna_dashboard/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/app.py b/optuna_dashboard/app.py index d7891a42..c6abf7a5 100644 --- a/optuna_dashboard/app.py +++ b/optuna_dashboard/app.py @@ -99,7 +99,7 @@ def get_distribution_name(param_name: str, study: Study) -> str: for trial in study.trials: if param_name in trial.distributions: return trial.distributions[param_name].__class__.__name__ - assert False + assert False, "Must not reach here." def create_app(storage: BaseStorage) -> Bottle: From 1de010590b5e564adc0365f7c6b1d9e07c3dd7ae Mon Sep 17 00:00:00 2001 From: Huzi Cheng Date: Thu, 11 Mar 2021 18:32:30 -0500 Subject: [PATCH 08/18] Update optuna_dashboard/app.py Co-authored-by: Masashi Shibata --- optuna_dashboard/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/app.py b/optuna_dashboard/app.py index c6abf7a5..264db499 100644 --- a/optuna_dashboard/app.py +++ b/optuna_dashboard/app.py @@ -196,7 +196,7 @@ def create_app(storage: BaseStorage) -> Bottle: @app.get("/api/studies//param_importances") @handle_json_api_exception def get_param_importances(study_id: int) -> BottleViewReturn: - # TODO: add support for selecting params and targets via query parameters. + # TODO(chenghuzi): add support for selecting params and targets via query parameters. response.content_type = "application/json" study_name = storage.get_study_name_from_id(study_id) study = Study(study_name=study_name, storage=storage) From b3fde3384401d945b7dc22cd2b6ba9ee2da25a70 Mon Sep 17 00:00:00 2001 From: Huzi Cheng Date: Thu, 11 Mar 2021 18:39:13 -0500 Subject: [PATCH 09/18] Update optuna_dashboard/static/apiClient.ts Co-authored-by: Masashi Shibata --- optuna_dashboard/static/apiClient.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/optuna_dashboard/static/apiClient.ts b/optuna_dashboard/static/apiClient.ts index 0e923dca..b3f888c4 100644 --- a/optuna_dashboard/static/apiClient.ts +++ b/optuna_dashboard/static/apiClient.ts @@ -186,5 +186,3 @@ export const getParamImportances = ( return res.data }) } - -export default getParamImportances From 64ddb452c86f2b43bc29da132507592bee6ca11c Mon Sep 17 00:00:00 2001 From: Cheng Huzi Date: Thu, 11 Mar 2021 18:49:24 -0500 Subject: [PATCH 10/18] Update install_requires --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index cb21a27d..dc10a76f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ install_requires = optuna>=2.4 bottle typing-extensions;python_version<'3.8' + scikit-learn [options.extras_require] lint = From 97422c63fcf2a8c617f076cf67885ede58f6e3b4 Mon Sep 17 00:00:00 2001 From: Henry Cui Date: Sat, 13 Mar 2021 15:51:23 +0900 Subject: [PATCH 11/18] Make axis selectable --- .../static/components/GraphParetoFront.tsx | 88 +++++++++++++++++-- 1 file changed, 79 insertions(+), 9 deletions(-) diff --git a/optuna_dashboard/static/components/GraphParetoFront.tsx b/optuna_dashboard/static/components/GraphParetoFront.tsx index f2a6f0cc..e0d4491b 100644 --- a/optuna_dashboard/static/components/GraphParetoFront.tsx +++ b/optuna_dashboard/static/components/GraphParetoFront.tsx @@ -1,21 +1,91 @@ import * as plotly from "plotly.js-dist" -import React, { FC, useEffect } from "react" +import React, { FC, useEffect, useState } from "react" +import { + Grid, + FormControl, + FormLabel, + MenuItem, + Select, +} from "@material-ui/core" +import { createStyles, makeStyles, Theme } from "@material-ui/core/styles" const plotDomId = "graph-pareto-front" +const useStyles = makeStyles((theme: Theme) => + createStyles({ + formControl: { + marginBottom: theme.spacing(2), + marginRight: theme.spacing(5), + marginTop: theme.spacing(10), + }, + }) +) + export const GraphParetoFront: FC<{ study: StudyDetail | null }> = ({ study = null }) => { + const classes = useStyles() + const [objectiveXId, setObjectiveXId] = useState(0) + const [objectiveYId, setObjectiveYId] = useState(1) + + const handleObjectiveXChange = ( + event: React.ChangeEvent<{ value: unknown }> + ) => { + setObjectiveXId(event.target.value as number) + } + + const handleObjectiveYChange = ( + event: React.ChangeEvent<{ value: unknown }> + ) => { + setObjectiveYId(event.target.value as number) + } + useEffect(() => { if (study != null) { - plotParetoFront(study) + plotParetoFront(study, objectiveXId, objectiveYId) } - }, [study]) + }, [study, objectiveXId, objectiveYId]) - return
+ return ( + + {study !== null && study.directions.length !== 1 ? ( + + + + Objective X ID: + + + + Objective Y ID: + + + + + ) : null} + +
+ + + ) } -const plotParetoFront = (study: StudyDetail) => { +const plotParetoFront = ( + study: StudyDetail, + objectiveXId: number, + objectiveYId: number +) => { if (document.getElementById(plotDomId) === null) { return } @@ -75,14 +145,14 @@ const plotParetoFront = (study: StudyDetail) => { { type: "scatter", x: completedTrials.map((t: Trial): number => { - return t.values![0] + return t.values![objectiveXId] }), y: completedTrials.map((t: Trial): number => { - return t.values![1] + return t.values![objectiveYId] }), mode: "markers", - xaxis: "Objective 0", - yaxis: "Objective 1", + xaxis: "Objective X", + yaxis: "Objective Y", marker: { color: pointColors, }, From b313f8cc20ed013b151030d4f47535561ccc70c8 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 14 Mar 2021 00:54:37 +0900 Subject: [PATCH 12/18] Implement search space cache --- optuna_dashboard/search_space.py | 68 +++++++++++++++++ tests/test_search_space.py | 124 +++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 optuna_dashboard/search_space.py create mode 100644 tests/test_search_space.py diff --git a/optuna_dashboard/search_space.py b/optuna_dashboard/search_space.py new file mode 100644 index 00000000..cd5464c2 --- /dev/null +++ b/optuna_dashboard/search_space.py @@ -0,0 +1,68 @@ +import copy +import threading +from typing import Dict, List, Optional, Set, Tuple + +from optuna.distributions import BaseDistribution +from optuna.study import BaseStudy +from optuna.trial import TrialState + +SearchSpaceSetT = Set[Tuple[str, BaseDistribution]] +SearchSpaceListT = List[Tuple[str, BaseDistribution]] + +# In-memory search space cache +search_space_cache_lock = threading.Lock() +search_space_cache: Dict[int, "_SearchSpace"] = {} + +states_of_interest = [TrialState.COMPLETE, TrialState.PRUNED] + + +def get_search_space(study: BaseStudy) -> Tuple[SearchSpaceListT, SearchSpaceListT]: + with search_space_cache_lock: + search_space = search_space_cache.get(study._study_id, None) + if search_space is None: + search_space = _SearchSpace() + search_space.update(study) + search_space_cache[study._study_id] = search_space + return search_space.intersection, search_space.union + + +class _SearchSpace: + def __init__(self) -> None: + self._cursor: int = -1 + self._intersection: Optional[SearchSpaceSetT] = None + self._union: SearchSpaceSetT = set() + + @property + def intersection(self) -> SearchSpaceListT: + if self._intersection is None: + return [] + intersection = list(self._intersection) + intersection.sort(key=lambda x: x[0]) + return intersection + + @property + def union(self) -> SearchSpaceListT: + union = list(self._union) + union.sort(key=lambda x: x[0]) + return union + + def update(self, study: BaseStudy) -> None: + next_cursor = self._cursor + for trial in reversed(study.get_trials(deepcopy=False)): + if self._cursor > trial.number: + break + + if not trial.state.is_finished(): + next_cursor = trial.number + + if trial.state not in states_of_interest: + continue + + current = set([(n, d) for n, d in trial.distributions.items()]) + self._union = self._union.union(current) + + if self._intersection is None: + self._intersection = copy.copy(current) + else: + self._intersection = self._intersection.intersection(current) + self._cursor = next_cursor diff --git a/tests/test_search_space.py b/tests/test_search_space.py new file mode 100644 index 00000000..15f1452a --- /dev/null +++ b/tests/test_search_space.py @@ -0,0 +1,124 @@ +import warnings +from unittest import TestCase + +import optuna +from optuna import create_trial +from optuna.distributions import UniformDistribution +from optuna.exceptions import ExperimentalWarning +from optuna.trial import TrialState + +from optuna_dashboard.search_space import _SearchSpace + + +class SearchSpaceTestCase(TestCase): + def setUp(self) -> None: + optuna.logging.set_verbosity(optuna.logging.ERROR) + warnings.simplefilter("ignore", category=ExperimentalWarning) + + def test_same_distributions(self) -> None: + study = optuna.create_study() + distributions = [ + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + ] + params = [ + { + "x0": 0.5, + "x1": 0.5, + }, + { + "x0": 0.5, + "x1": 0.5, + }, + ] + trials = [ + create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p) + for d, p in zip(distributions, params) + ] + study.add_trials(trials=trials) + + search_space = _SearchSpace() + search_space.update(study) + + self.assertEqual(len(search_space.intersection), 2) + self.assertEqual(len(search_space.union), 2) + + def test_different_distributions(self) -> None: + study = optuna.create_study() + distributions = [ + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + { + "x0": UniformDistribution(low=0, high=5), + "x1": UniformDistribution(low=0, high=10), + }, + ] + params = [ + { + "x0": 0.5, + "x1": 0.5, + }, + { + "x0": 0.5, + "x1": 0.5, + }, + ] + trials = [ + create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p) + for d, p in zip(distributions, params) + ] + study.add_trials(trials=trials) + + search_space = _SearchSpace() + search_space.update(study) + + self.assertEqual(len(search_space.intersection), 1) + self.assertEqual(len(search_space.union), 3) + + def test_dynamic_search_space(self) -> None: + study = optuna.create_study() + distributions = [ + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + { + "x0": UniformDistribution(low=0, high=5), + }, + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + ] + params = [ + { + "x0": 0.5, + "x1": 0.5, + }, + { + "x0": 0.5, + }, + { + "x0": 0.5, + "x1": 0.5, + }, + ] + trials = [ + create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p) + for d, p in zip(distributions, params) + ] + study.add_trials(trials=trials) + + search_space = _SearchSpace() + search_space.update(study) + + self.assertEqual(len(search_space.intersection), 0) + self.assertEqual(len(search_space.union), 3) From bd7af41f58f4612206442ddb5b7117ee855b0c64 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 14 Mar 2021 01:17:26 +0900 Subject: [PATCH 13/18] Change interface for search space --- optuna_dashboard/search_space.py | 15 +++++++-------- tests/test_search_space.py | 15 +++------------ 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/optuna_dashboard/search_space.py b/optuna_dashboard/search_space.py index cd5464c2..bafd0273 100644 --- a/optuna_dashboard/search_space.py +++ b/optuna_dashboard/search_space.py @@ -3,8 +3,7 @@ import threading from typing import Dict, List, Optional, Set, Tuple from optuna.distributions import BaseDistribution -from optuna.study import BaseStudy -from optuna.trial import TrialState +from optuna.trial import TrialState, FrozenTrial SearchSpaceSetT = Set[Tuple[str, BaseDistribution]] SearchSpaceListT = List[Tuple[str, BaseDistribution]] @@ -16,13 +15,13 @@ search_space_cache: Dict[int, "_SearchSpace"] = {} states_of_interest = [TrialState.COMPLETE, TrialState.PRUNED] -def get_search_space(study: BaseStudy) -> Tuple[SearchSpaceListT, SearchSpaceListT]: +def get_search_space(study_id: int, trials: List[FrozenTrial]) -> Tuple[SearchSpaceListT, SearchSpaceListT]: with search_space_cache_lock: - search_space = search_space_cache.get(study._study_id, None) + search_space = search_space_cache.get(study_id, None) if search_space is None: search_space = _SearchSpace() - search_space.update(study) - search_space_cache[study._study_id] = search_space + search_space.update(trials) + search_space_cache[study_id] = search_space return search_space.intersection, search_space.union @@ -46,9 +45,9 @@ class _SearchSpace: union.sort(key=lambda x: x[0]) return union - def update(self, study: BaseStudy) -> None: + def update(self, trials: List[FrozenTrial]) -> None: next_cursor = self._cursor - for trial in reversed(study.get_trials(deepcopy=False)): + for trial in reversed(trials): if self._cursor > trial.number: break diff --git a/tests/test_search_space.py b/tests/test_search_space.py index 15f1452a..61a34e63 100644 --- a/tests/test_search_space.py +++ b/tests/test_search_space.py @@ -16,7 +16,6 @@ class SearchSpaceTestCase(TestCase): warnings.simplefilter("ignore", category=ExperimentalWarning) def test_same_distributions(self) -> None: - study = optuna.create_study() distributions = [ { "x0": UniformDistribution(low=0, high=10), @@ -41,16 +40,13 @@ class SearchSpaceTestCase(TestCase): create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p) for d, p in zip(distributions, params) ] - study.add_trials(trials=trials) - search_space = _SearchSpace() - search_space.update(study) + search_space.update(trials) self.assertEqual(len(search_space.intersection), 2) self.assertEqual(len(search_space.union), 2) def test_different_distributions(self) -> None: - study = optuna.create_study() distributions = [ { "x0": UniformDistribution(low=0, high=10), @@ -75,16 +71,13 @@ class SearchSpaceTestCase(TestCase): create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p) for d, p in zip(distributions, params) ] - study.add_trials(trials=trials) - search_space = _SearchSpace() - search_space.update(study) + search_space.update(trials) self.assertEqual(len(search_space.intersection), 1) self.assertEqual(len(search_space.union), 3) def test_dynamic_search_space(self) -> None: - study = optuna.create_study() distributions = [ { "x0": UniformDistribution(low=0, high=10), @@ -115,10 +108,8 @@ class SearchSpaceTestCase(TestCase): create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p) for d, p in zip(distributions, params) ] - study.add_trials(trials=trials) - search_space = _SearchSpace() - search_space.update(study) + search_space.update(trials) self.assertEqual(len(search_space.intersection), 0) self.assertEqual(len(search_space.union), 3) From 33e15578ac84c427a612717609df60c6a7743e4e Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 14 Mar 2021 01:25:12 +0900 Subject: [PATCH 14/18] Return intersection/union search space --- optuna_dashboard/app.py | 4 +++- optuna_dashboard/search_space.py | 4 +++- optuna_dashboard/serializer.py | 26 ++++++++++++++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/optuna_dashboard/app.py b/optuna_dashboard/app.py index e1af9ca9..04a3e4f0 100644 --- a/optuna_dashboard/app.py +++ b/optuna_dashboard/app.py @@ -14,6 +14,7 @@ from optuna.trial import FrozenTrial from optuna.study import StudyDirection, StudySummary from . import serializer +from .search_space import get_search_space BottleViewReturn = Union[str, bytes, Dict[str, Any], BaseResponse] BottleView = TypeVar("BottleView", bound=Callable[..., BottleViewReturn]) @@ -183,7 +184,8 @@ def create_app(storage: BaseStorage) -> Bottle: response.status = 404 # Not found return {"reason": f"study_id={study_id} is not found"} trials = get_trials(storage, study_id) - return serializer.serialize_study_detail(summary, trials) + intersection, union = get_search_space(study_id, trials) + return serializer.serialize_study_detail(summary, trials, intersection, union) @app.get("/static/") def send_static(filename: str) -> BottleViewReturn: diff --git a/optuna_dashboard/search_space.py b/optuna_dashboard/search_space.py index bafd0273..1f740fde 100644 --- a/optuna_dashboard/search_space.py +++ b/optuna_dashboard/search_space.py @@ -15,7 +15,9 @@ search_space_cache: Dict[int, "_SearchSpace"] = {} states_of_interest = [TrialState.COMPLETE, TrialState.PRUNED] -def get_search_space(study_id: int, trials: List[FrozenTrial]) -> Tuple[SearchSpaceListT, SearchSpaceListT]: +def get_search_space( + study_id: int, trials: List[FrozenTrial] +) -> Tuple[SearchSpaceListT, SearchSpaceListT]: with search_space_cache_lock: search_space = search_space_cache.get(study_id, None) if search_space is None: diff --git a/optuna_dashboard/serializer.py b/optuna_dashboard/serializer.py index 76eef2f4..e164ff78 100644 --- a/optuna_dashboard/serializer.py +++ b/optuna_dashboard/serializer.py @@ -1,6 +1,7 @@ import json -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple +from optuna.distributions import BaseDistribution from optuna.study import StudySummary from optuna.trial import FrozenTrial @@ -66,7 +67,10 @@ def serialize_study_summary(summary: StudySummary) -> Dict[str, Any]: def serialize_study_detail( - summary: StudySummary, trials: List[FrozenTrial] + summary: StudySummary, + trials: List[FrozenTrial], + intersection: List[Tuple[str, BaseDistribution]], + union: List[Tuple[str, BaseDistribution]], ) -> Dict[str, Any]: serialized: Dict[str, Any] = { "name": summary.study_name, @@ -83,6 +87,9 @@ def serialize_study_detail( serialized["trials"] = [ serialize_frozen_trial(summary._study_id, trial) for trial in trials ] + + serialized["intersection_search_space"] = serialize_search_space(intersection) + serialized["union_search_space"] = serialize_search_space(union) return serialized @@ -108,3 +115,18 @@ def serialize_frozen_trial(study_id: int, trial: FrozenTrial) -> Dict[str, Any]: serialized["datetime_complete"] = trial.datetime_complete.isoformat() return serialized + + +def serialize_search_space( + search_space: List[Tuple[str, BaseDistribution]] +) -> List[Dict[str, Any]]: + serialized = [] + for param_name, distribution in search_space: + serialized.append( + { + "name": param_name, + "type": distribution.__class__.__name__, + "attributes": distribution._asdict(), + } + ) + return serialized From 62dbcb0fb844ef5b015d96811d49cb192d76df5e Mon Sep 17 00:00:00 2001 From: Huzi Cheng Date: Sat, 13 Mar 2021 13:15:13 -0500 Subject: [PATCH 15/18] Update optuna_dashboard/static/components/HyperparameterImportances.tsx Co-authored-by: Masashi Shibata --- optuna_dashboard/static/components/HyperparameterImportances.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/optuna_dashboard/static/components/HyperparameterImportances.tsx b/optuna_dashboard/static/components/HyperparameterImportances.tsx index 59904769..f5969345 100644 --- a/optuna_dashboard/static/components/HyperparameterImportances.tsx +++ b/optuna_dashboard/static/components/HyperparameterImportances.tsx @@ -51,7 +51,6 @@ const plotParamImportances = (paramsImportanceData: ParamImportances) => { const param_hover_templates = param_importances.map( (p) => `${p.name} (${p.distribution}): ${p.importance} ` ) - console.log("param_colors", param_colors) const layout: Partial = { title: "Hyperparameter Importance", From 108c939d73c3f80769143827c97addff6dac3d56 Mon Sep 17 00:00:00 2001 From: Huzi Cheng Date: Sat, 13 Mar 2021 13:22:52 -0500 Subject: [PATCH 16/18] Update optuna_dashboard/app.py Co-authored-by: Masashi Shibata --- optuna_dashboard/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/optuna_dashboard/app.py b/optuna_dashboard/app.py index 264db499..542b7c0f 100644 --- a/optuna_dashboard/app.py +++ b/optuna_dashboard/app.py @@ -217,11 +217,11 @@ def create_app(storage: BaseStorage) -> Bottle: "target_name": target_name, "param_importances": [ { - "name": i[0], - "importance": i[1], - "distribution": get_distribution_name(i[0], study), + "name": name, + "importance": importance, + "distribution": get_distribution_name(name, study), } - for i in importances.items() + for name, importance in importances.items() ], } From 468bed28987cb4bff657afda810b3c71a9f6873a Mon Sep 17 00:00:00 2001 From: Cheng Huzi Date: Sat, 13 Mar 2021 14:18:09 -0500 Subject: [PATCH 17/18] Add dependency for updating parameter importance --- .../static/components/HyperparameterImportances.tsx | 6 +++--- optuna_dashboard/static/components/StudyDetail.tsx | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/optuna_dashboard/static/components/HyperparameterImportances.tsx b/optuna_dashboard/static/components/HyperparameterImportances.tsx index f5969345..1af1c1ea 100644 --- a/optuna_dashboard/static/components/HyperparameterImportances.tsx +++ b/optuna_dashboard/static/components/HyperparameterImportances.tsx @@ -26,15 +26,15 @@ const distributionColors = { } export const HyperparameterImportances: FC<{ - studyId: number -}> = ({ studyId }) => { + studyId: number, numOfTrials: number +}> = ({ studyId, numOfTrials = 0 }) => { useEffect(() => { async function fetchAndPlotParamImportances(studyId: number) { const paramsImportanceData = await getParamImportances(studyId) plotParamImportances(paramsImportanceData) } fetchAndPlotParamImportances(studyId) - }) + }, [numOfTrials]) return
} diff --git a/optuna_dashboard/static/components/StudyDetail.tsx b/optuna_dashboard/static/components/StudyDetail.tsx index 56e84507..154a6bf0 100644 --- a/optuna_dashboard/static/components/StudyDetail.tsx +++ b/optuna_dashboard/static/components/StudyDetail.tsx @@ -202,7 +202,7 @@ export const StudyDetail: FC = () => { - + From be2af8ea9142c0a2763b20250a4427b85776d1a6 Mon Sep 17 00:00:00 2001 From: Cheng Huzi Date: Sat, 13 Mar 2021 14:56:12 -0500 Subject: [PATCH 18/18] Format code --- .../static/components/HyperparameterImportances.tsx | 3 ++- optuna_dashboard/static/components/StudyDetail.tsx | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/static/components/HyperparameterImportances.tsx b/optuna_dashboard/static/components/HyperparameterImportances.tsx index 1af1c1ea..99280bf6 100644 --- a/optuna_dashboard/static/components/HyperparameterImportances.tsx +++ b/optuna_dashboard/static/components/HyperparameterImportances.tsx @@ -26,7 +26,8 @@ const distributionColors = { } export const HyperparameterImportances: FC<{ - studyId: number, numOfTrials: number + studyId: number + numOfTrials: number }> = ({ studyId, numOfTrials = 0 }) => { useEffect(() => { async function fetchAndPlotParamImportances(studyId: number) { diff --git a/optuna_dashboard/static/components/StudyDetail.tsx b/optuna_dashboard/static/components/StudyDetail.tsx index 154a6bf0..2b42afca 100644 --- a/optuna_dashboard/static/components/StudyDetail.tsx +++ b/optuna_dashboard/static/components/StudyDetail.tsx @@ -202,7 +202,10 @@ export const StudyDetail: FC = () => { - +