diff --git a/optuna_dashboard/app.py b/optuna_dashboard/app.py index e1af9ca9..97af477e 100644 --- a/optuna_dashboard/app.py +++ b/optuna_dashboard/app.py @@ -8,12 +8,14 @@ import traceback from typing import Union, Dict, List, Optional, TypeVar, Callable, Any, cast from bottle import Bottle, BaseResponse, redirect, request, response, static_file +import optuna from optuna.exceptions import DuplicatedStudyError from optuna.storages import BaseStorage -from optuna.trial import FrozenTrial -from optuna.study import StudyDirection, StudySummary +from optuna.trial import FrozenTrial, TrialState +from optuna.study import StudyDirection, StudySummary, Study from . import serializer +from .search_space import get_search_space BottleViewReturn = Union[str, bytes, Dict[str, Any], BaseResponse] BottleView = TypeVar("BottleView", bound=Callable[..., BottleViewReturn]) @@ -94,6 +96,13 @@ def get_trials( return trials +def get_distribution_name(param_name: str, study: Study) -> str: + for trial in study.trials: + if param_name in trial.distributions: + return trial.distributions[param_name].__class__.__name__ + assert False, "Must not reach here." + + def create_app(storage: BaseStorage) -> Bottle: app = Bottle() @@ -183,7 +192,40 @@ def create_app(storage: BaseStorage) -> Bottle: response.status = 404 # Not found return {"reason": f"study_id={study_id} is not found"} trials = get_trials(storage, study_id) - return serializer.serialize_study_detail(summary, trials) + intersection, union = get_search_space(study_id, trials) + return serializer.serialize_study_detail(summary, trials, intersection, union) + + @app.get("/api/studies//param_importances") + @handle_json_api_exception + def get_param_importances(study_id: int) -> BottleViewReturn: + # TODO(chenghuzi): add support for selecting params and targets via query parameters. + response.content_type = "application/json" + study_name = storage.get_study_name_from_id(study_id) + study = Study(study_name=study_name, storage=storage) + + trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE] + if len(trials) == 0: + return "" + evaluator = None + params = None + target = None + importances = optuna.importance.get_param_importances( + study, evaluator=evaluator, params=params, target=target + ) + if target is None: + target_name = "Objective Value" + + return { + "target_name": target_name, + "param_importances": [ + { + "name": name, + "importance": importance, + "distribution": get_distribution_name(name, study), + } + for name, importance in importances.items() + ], + } @app.get("/static/") def send_static(filename: str) -> BottleViewReturn: diff --git a/optuna_dashboard/search_space.py b/optuna_dashboard/search_space.py new file mode 100644 index 00000000..1f740fde --- /dev/null +++ b/optuna_dashboard/search_space.py @@ -0,0 +1,69 @@ +import copy +import threading +from typing import Dict, List, Optional, Set, Tuple + +from optuna.distributions import BaseDistribution +from optuna.trial import TrialState, FrozenTrial + +SearchSpaceSetT = Set[Tuple[str, BaseDistribution]] +SearchSpaceListT = List[Tuple[str, BaseDistribution]] + +# In-memory search space cache +search_space_cache_lock = threading.Lock() +search_space_cache: Dict[int, "_SearchSpace"] = {} + +states_of_interest = [TrialState.COMPLETE, TrialState.PRUNED] + + +def get_search_space( + study_id: int, trials: List[FrozenTrial] +) -> Tuple[SearchSpaceListT, SearchSpaceListT]: + with search_space_cache_lock: + search_space = search_space_cache.get(study_id, None) + if search_space is None: + search_space = _SearchSpace() + search_space.update(trials) + search_space_cache[study_id] = search_space + return search_space.intersection, search_space.union + + +class _SearchSpace: + def __init__(self) -> None: + self._cursor: int = -1 + self._intersection: Optional[SearchSpaceSetT] = None + self._union: SearchSpaceSetT = set() + + @property + def intersection(self) -> SearchSpaceListT: + if self._intersection is None: + return [] + intersection = list(self._intersection) + intersection.sort(key=lambda x: x[0]) + return intersection + + @property + def union(self) -> SearchSpaceListT: + union = list(self._union) + union.sort(key=lambda x: x[0]) + return union + + def update(self, trials: List[FrozenTrial]) -> None: + next_cursor = self._cursor + for trial in reversed(trials): + if self._cursor > trial.number: + break + + if not trial.state.is_finished(): + next_cursor = trial.number + + if trial.state not in states_of_interest: + continue + + current = set([(n, d) for n, d in trial.distributions.items()]) + self._union = self._union.union(current) + + if self._intersection is None: + self._intersection = copy.copy(current) + else: + self._intersection = self._intersection.intersection(current) + self._cursor = next_cursor diff --git a/optuna_dashboard/serializer.py b/optuna_dashboard/serializer.py index 76eef2f4..e164ff78 100644 --- a/optuna_dashboard/serializer.py +++ b/optuna_dashboard/serializer.py @@ -1,6 +1,7 @@ import json -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple +from optuna.distributions import BaseDistribution from optuna.study import StudySummary from optuna.trial import FrozenTrial @@ -66,7 +67,10 @@ def serialize_study_summary(summary: StudySummary) -> Dict[str, Any]: def serialize_study_detail( - summary: StudySummary, trials: List[FrozenTrial] + summary: StudySummary, + trials: List[FrozenTrial], + intersection: List[Tuple[str, BaseDistribution]], + union: List[Tuple[str, BaseDistribution]], ) -> Dict[str, Any]: serialized: Dict[str, Any] = { "name": summary.study_name, @@ -83,6 +87,9 @@ def serialize_study_detail( serialized["trials"] = [ serialize_frozen_trial(summary._study_id, trial) for trial in trials ] + + serialized["intersection_search_space"] = serialize_search_space(intersection) + serialized["union_search_space"] = serialize_search_space(union) return serialized @@ -108,3 +115,18 @@ def serialize_frozen_trial(study_id: int, trial: FrozenTrial) -> Dict[str, Any]: serialized["datetime_complete"] = trial.datetime_complete.isoformat() return serialized + + +def serialize_search_space( + search_space: List[Tuple[str, BaseDistribution]] +) -> List[Dict[str, Any]]: + serialized = [] + for param_name, distribution in search_space: + serialized.append( + { + "name": param_name, + "type": distribution.__class__.__name__, + "attributes": distribution._asdict(), + } + ) + return serialized diff --git a/optuna_dashboard/static/apiClient.ts b/optuna_dashboard/static/apiClient.ts index cf9348f9..b3f888c4 100644 --- a/optuna_dashboard/static/apiClient.ts +++ b/optuna_dashboard/static/apiClient.ts @@ -168,3 +168,21 @@ export const deleteStudyAPI = (studyId: number) => { return {} }) } + +interface ParamImportancesResponse { + target_name: string + param_importances: ParamImportance[] +} + +export const getParamImportances = ( + studyId: number +): Promise => { + return axiosInstance + .get( + `/api/studies/${studyId}/param_importances`, + {} + ) + .then((res) => { + return res.data + }) +} diff --git a/optuna_dashboard/static/components/GraphParetoFront.tsx b/optuna_dashboard/static/components/GraphParetoFront.tsx new file mode 100644 index 00000000..e0d4491b --- /dev/null +++ b/optuna_dashboard/static/components/GraphParetoFront.tsx @@ -0,0 +1,175 @@ +import * as plotly from "plotly.js-dist" +import React, { FC, useEffect, useState } from "react" +import { + Grid, + FormControl, + FormLabel, + MenuItem, + Select, +} from "@material-ui/core" +import { createStyles, makeStyles, Theme } from "@material-ui/core/styles" + +const plotDomId = "graph-pareto-front" + +const useStyles = makeStyles((theme: Theme) => + createStyles({ + formControl: { + marginBottom: theme.spacing(2), + marginRight: theme.spacing(5), + marginTop: theme.spacing(10), + }, + }) +) + +export const GraphParetoFront: FC<{ + study: StudyDetail | null +}> = ({ study = null }) => { + const classes = useStyles() + const [objectiveXId, setObjectiveXId] = useState(0) + const [objectiveYId, setObjectiveYId] = useState(1) + + const handleObjectiveXChange = ( + event: React.ChangeEvent<{ value: unknown }> + ) => { + setObjectiveXId(event.target.value as number) + } + + const handleObjectiveYChange = ( + event: React.ChangeEvent<{ value: unknown }> + ) => { + setObjectiveYId(event.target.value as number) + } + + useEffect(() => { + if (study != null) { + plotParetoFront(study, objectiveXId, objectiveYId) + } + }, [study, objectiveXId, objectiveYId]) + + return ( + + {study !== null && study.directions.length !== 1 ? ( + + + + Objective X ID: + + + + Objective Y ID: + + + + + ) : null} + +
+ + + ) +} + +const plotParetoFront = ( + study: StudyDetail, + objectiveXId: number, + objectiveYId: number +) => { + if (document.getElementById(plotDomId) === null) { + return + } + + const dim: number = study.directions.length + if (dim != 2) { + return + } + const layout: Partial = { + title: "Pareto-front plot", + margin: { + l: 50, + r: 50, + b: 0, + }, + } + + const trials: Trial[] = study !== null ? study.trials : [] + const completedTrials = trials.filter((t) => t.state === "Complete") + + if (completedTrials.length === 0) { + plotly.react(plotDomId, [], layout) + return + } + + const normalizedValues: number[][] = [] + completedTrials.forEach((t) => { + if (t.values && t.values.length == dim) { + const trialValues = t.values.map((v: number, i: number) => { + return study.directions[i] === "minimize" ? v : -v + }) + normalizedValues.push(trialValues) + } + }) + + const pointColors: string[] = [] + normalizedValues.forEach((values0: number[], i: number) => { + let dominated = false + + dominated = normalizedValues.some((values1: number[], j: number) => { + if (i === j) { + return false + } + return values0.every((value0: number, k: number) => { + return values1[k] <= value0 + }) + }) + + if (dominated) { + pointColors.push("blue") + } else { + pointColors.push("red") + } + }) + + const plotData: Partial[] = [ + { + type: "scatter", + x: completedTrials.map((t: Trial): number => { + return t.values![objectiveXId] + }), + y: completedTrials.map((t: Trial): number => { + return t.values![objectiveYId] + }), + mode: "markers", + xaxis: "Objective X", + yaxis: "Objective Y", + marker: { + color: pointColors, + }, + text: completedTrials.map((t: Trial): string => { + return JSON.stringify( + { + number: t.number, + values: t.values, + params: t.params, + }, + null, + 2 + ).replaceAll("\n", "
") + }), + hovertemplate: "%{text}", + }, + ] + + plotly.react(plotDomId, plotData, layout) +} diff --git a/optuna_dashboard/static/components/HyperparameterImportances.tsx b/optuna_dashboard/static/components/HyperparameterImportances.tsx new file mode 100644 index 00000000..99280bf6 --- /dev/null +++ b/optuna_dashboard/static/components/HyperparameterImportances.tsx @@ -0,0 +1,88 @@ +import * as plotly from "plotly.js-dist" +import React, { FC, useEffect } from "react" +import { getParamImportances } from "../apiClient" +const plotDomId = "graph-hyperparameter-importances" + +// To match colors used by plot_param_importances in optuna. +const plotlyColorsSequentialBlues = [ + "rgb(247,251,255)", + "rgb(222,235,247)", + "rgb(198,219,239)", + "rgb(158,202,225)", + "rgb(107,174,214)", + "rgb(66,146,198)", + "rgb(33,113,181)", + "rgb(8,81,156)", + "rgb(8,48,107)", +] + +const distributionColors = { + UniformDistribution: plotlyColorsSequentialBlues.slice(-1)[0], + LogUniformDistribution: plotlyColorsSequentialBlues.slice(-1)[0], + DiscreteUniformDistribution: plotlyColorsSequentialBlues.slice(-1)[0], + IntUniformDistribution: plotlyColorsSequentialBlues.slice(-2)[0], + IntLogUniformDistribution: plotlyColorsSequentialBlues.slice(-2)[0], + CategoricalDistribution: plotlyColorsSequentialBlues.slice(-4)[0], +} + +export const HyperparameterImportances: FC<{ + studyId: number + numOfTrials: number +}> = ({ studyId, numOfTrials = 0 }) => { + useEffect(() => { + async function fetchAndPlotParamImportances(studyId: number) { + const paramsImportanceData = await getParamImportances(studyId) + plotParamImportances(paramsImportanceData) + } + fetchAndPlotParamImportances(studyId) + }, [numOfTrials]) + return
+} + +const plotParamImportances = (paramsImportanceData: ParamImportances) => { + if (document.getElementById(plotDomId) === null) { + return + } + const param_importances = paramsImportanceData.param_importances.reverse() + const importance_values = param_importances.map((p) => p.importance) + const param_names = param_importances.map((p) => p.name) + const param_colors = param_importances.map( + (p) => distributionColors[p.distribution] + ) + const param_hover_templates = param_importances.map( + (p) => `${p.name} (${p.distribution}): ${p.importance} ` + ) + + const layout: Partial = { + title: "Hyperparameter Importance", + xaxis: { + title: `Importance for ${paramsImportanceData.target_name}`, + }, + yaxis: { + title: "Hyperparameter", + }, + margin: { + l: 50, + r: 50, + b: 50, + }, + showlegend: false, + } + + const plotData: Partial[] = [ + { + type: "bar", + orientation: "h", + x: importance_values, + y: param_names, + text: importance_values.map((v) => String(v.toFixed(2))), + textposition: "outside", + hovertemplate: param_hover_templates, + marker: { + color: param_colors, + }, + }, + ] + + plotly.react(plotDomId, plotData, layout) +} diff --git a/optuna_dashboard/static/components/StudyDetail.tsx b/optuna_dashboard/static/components/StudyDetail.tsx index aeaad06d..b46c9bd7 100644 --- a/optuna_dashboard/static/components/StudyDetail.tsx +++ b/optuna_dashboard/static/components/StudyDetail.tsx @@ -20,10 +20,12 @@ import { Home, Cached } from "@material-ui/icons" import { DataGridColumn, DataGrid } from "./DataGrid" import { GraphParallelCoordinate } from "./GraphParallelCoordinate" +import { HyperparameterImportances } from "./HyperparameterImportances" import { GraphIntermediateValues } from "./GraphIntermediateValues" import { GraphSlice } from "./GraphSlice" import { GraphHistory } from "./GraphHistory" import { GraphContour } from "./GraphContour" +import { GraphParetoFront } from "./GraphParetoFront" import { actionCreator } from "../action" import { studyDetailsState } from "../state" @@ -197,6 +199,20 @@ export const StudyDetail: FC = () => { ) : null} + {studyDetail !== null && isSingleObjectiveStudy(studyDetail) ? ( + + + + + + + + + + ) : null} {studyDetail !== null ? ( @@ -210,6 +226,13 @@ export const StudyDetail: FC = () => { + ): null} + {studyDetail !== null && !isSingleObjectiveStudy(studyDetail) ? ( + + + + + ) : null} diff --git a/optuna_dashboard/static/types/index.d.ts b/optuna_dashboard/static/types/index.d.ts index cc320769..2ae3ee30 100644 --- a/optuna_dashboard/static/types/index.d.ts +++ b/optuna_dashboard/static/types/index.d.ts @@ -9,6 +9,13 @@ declare const URL_PREFIX: string type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type StudyDirection = "maximize" | "minimize" | "not_set" +type Distribution = + | "UniformDistribution" + | "LogUniformDistribution" + | "DiscreteUniformDistribution" + | "IntUniformDistribution" + | "IntLogUniformDistribution" + | "CategoricalDistribution" declare interface TrialIntermediateValue { step: number @@ -20,6 +27,12 @@ declare interface TrialParam { value: string } +declare interface ParamImportance { + name: string + importance: number + distribution: Distribution +} + declare interface Attribute { key: string value: string @@ -60,3 +73,8 @@ declare interface StudyDetail { declare interface StudyDetails { [study_id: string]: StudyDetail } + +declare interface ParamImportances { + target_name: string + param_importances: ParamImportance[] +} diff --git a/setup.cfg b/setup.cfg index cb21a27d..dc10a76f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ install_requires = optuna>=2.4 bottle typing-extensions;python_version<'3.8' + scikit-learn [options.extras_require] lint = diff --git a/tests/test_search_space.py b/tests/test_search_space.py new file mode 100644 index 00000000..61a34e63 --- /dev/null +++ b/tests/test_search_space.py @@ -0,0 +1,115 @@ +import warnings +from unittest import TestCase + +import optuna +from optuna import create_trial +from optuna.distributions import UniformDistribution +from optuna.exceptions import ExperimentalWarning +from optuna.trial import TrialState + +from optuna_dashboard.search_space import _SearchSpace + + +class SearchSpaceTestCase(TestCase): + def setUp(self) -> None: + optuna.logging.set_verbosity(optuna.logging.ERROR) + warnings.simplefilter("ignore", category=ExperimentalWarning) + + def test_same_distributions(self) -> None: + distributions = [ + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + ] + params = [ + { + "x0": 0.5, + "x1": 0.5, + }, + { + "x0": 0.5, + "x1": 0.5, + }, + ] + trials = [ + create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p) + for d, p in zip(distributions, params) + ] + search_space = _SearchSpace() + search_space.update(trials) + + self.assertEqual(len(search_space.intersection), 2) + self.assertEqual(len(search_space.union), 2) + + def test_different_distributions(self) -> None: + distributions = [ + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + { + "x0": UniformDistribution(low=0, high=5), + "x1": UniformDistribution(low=0, high=10), + }, + ] + params = [ + { + "x0": 0.5, + "x1": 0.5, + }, + { + "x0": 0.5, + "x1": 0.5, + }, + ] + trials = [ + create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p) + for d, p in zip(distributions, params) + ] + search_space = _SearchSpace() + search_space.update(trials) + + self.assertEqual(len(search_space.intersection), 1) + self.assertEqual(len(search_space.union), 3) + + def test_dynamic_search_space(self) -> None: + distributions = [ + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + { + "x0": UniformDistribution(low=0, high=5), + }, + { + "x0": UniformDistribution(low=0, high=10), + "x1": UniformDistribution(low=0, high=10), + }, + ] + params = [ + { + "x0": 0.5, + "x1": 0.5, + }, + { + "x0": 0.5, + }, + { + "x0": 0.5, + "x1": 0.5, + }, + ] + trials = [ + create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p) + for d, p in zip(distributions, params) + ] + search_space = _SearchSpace() + search_space.update(trials) + + self.assertEqual(len(search_space.intersection), 0) + self.assertEqual(len(search_space.union), 3)