Resolve Conflicts

optuna_dashboard/app.py

@@ -8,12 +8,14 @@ import traceback
 from typing import Union, Dict, List, Optional, TypeVar, Callable, Any, cast
 
 from bottle import Bottle, BaseResponse, redirect, request, response, static_file
+import optuna
 from optuna.exceptions import DuplicatedStudyError
 from optuna.storages import BaseStorage
-from optuna.trial import FrozenTrial
-from optuna.study import StudyDirection, StudySummary
+from optuna.trial import FrozenTrial, TrialState
+from optuna.study import StudyDirection, StudySummary, Study
 
 from . import serializer
+from .search_space import get_search_space
 
 BottleViewReturn = Union[str, bytes, Dict[str, Any], BaseResponse]
 BottleView = TypeVar("BottleView", bound=Callable[..., BottleViewReturn])
@@ -94,6 +96,13 @@ def get_trials(
     return trials
 
 
+def get_distribution_name(param_name: str, study: Study) -> str:
+    for trial in study.trials:
+        if param_name in trial.distributions:
+            return trial.distributions[param_name].__class__.__name__
+    assert False, "Must not reach here."
+
+
 def create_app(storage: BaseStorage) -> Bottle:
     app = Bottle()
 
@@ -183,7 +192,40 @@ def create_app(storage: BaseStorage) -> Bottle:
             response.status = 404  # Not found
             return {"reason": f"study_id={study_id} is not found"}
         trials = get_trials(storage, study_id)
-        return serializer.serialize_study_detail(summary, trials)
+        intersection, union = get_search_space(study_id, trials)
+        return serializer.serialize_study_detail(summary, trials, intersection, union)
+
+    @app.get("/api/studies/<study_id:int>/param_importances")
+    @handle_json_api_exception
+    def get_param_importances(study_id: int) -> BottleViewReturn:
+        # TODO(chenghuzi): add support for selecting params and targets via query parameters.
+        response.content_type = "application/json"
+        study_name = storage.get_study_name_from_id(study_id)
+        study = Study(study_name=study_name, storage=storage)
+
+        trials = [trial for trial in study.trials if trial.state == TrialState.COMPLETE]
+        if len(trials) == 0:
+            return ""
+        evaluator = None
+        params = None
+        target = None
+        importances = optuna.importance.get_param_importances(
+            study, evaluator=evaluator, params=params, target=target
+        )
+        if target is None:
+            target_name = "Objective Value"
+
+        return {
+            "target_name": target_name,
+            "param_importances": [
+                {
+                    "name": name,
+                    "importance": importance,
+                    "distribution": get_distribution_name(name, study),
+                }
+                for name, importance in importances.items()
+            ],
+        }
 
     @app.get("/static/<filename:path>")
     def send_static(filename: str) -> BottleViewReturn:

optuna_dashboard/search_space.py

@@ -0,0 +1,69 @@
+import copy
+import threading
+from typing import Dict, List, Optional, Set, Tuple
+
+from optuna.distributions import BaseDistribution
+from optuna.trial import TrialState, FrozenTrial
+
+SearchSpaceSetT = Set[Tuple[str, BaseDistribution]]
+SearchSpaceListT = List[Tuple[str, BaseDistribution]]
+
+# In-memory search space cache
+search_space_cache_lock = threading.Lock()
+search_space_cache: Dict[int, "_SearchSpace"] = {}
+
+states_of_interest = [TrialState.COMPLETE, TrialState.PRUNED]
+
+
+def get_search_space(
+    study_id: int, trials: List[FrozenTrial]
+) -> Tuple[SearchSpaceListT, SearchSpaceListT]:
+    with search_space_cache_lock:
+        search_space = search_space_cache.get(study_id, None)
+        if search_space is None:
+            search_space = _SearchSpace()
+        search_space.update(trials)
+        search_space_cache[study_id] = search_space
+        return search_space.intersection, search_space.union
+
+
+class _SearchSpace:
+    def __init__(self) -> None:
+        self._cursor: int = -1
+        self._intersection: Optional[SearchSpaceSetT] = None
+        self._union: SearchSpaceSetT = set()
+
+    @property
+    def intersection(self) -> SearchSpaceListT:
+        if self._intersection is None:
+            return []
+        intersection = list(self._intersection)
+        intersection.sort(key=lambda x: x[0])
+        return intersection
+
+    @property
+    def union(self) -> SearchSpaceListT:
+        union = list(self._union)
+        union.sort(key=lambda x: x[0])
+        return union
+
+    def update(self, trials: List[FrozenTrial]) -> None:
+        next_cursor = self._cursor
+        for trial in reversed(trials):
+            if self._cursor > trial.number:
+                break
+
+            if not trial.state.is_finished():
+                next_cursor = trial.number
+
+            if trial.state not in states_of_interest:
+                continue
+
+            current = set([(n, d) for n, d in trial.distributions.items()])
+            self._union = self._union.union(current)
+
+            if self._intersection is None:
+                self._intersection = copy.copy(current)
+            else:
+                self._intersection = self._intersection.intersection(current)
+        self._cursor = next_cursor

optuna_dashboard/serializer.py

@@ -1,6 +1,7 @@
 import json
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Tuple
 
+from optuna.distributions import BaseDistribution
 from optuna.study import StudySummary
 from optuna.trial import FrozenTrial
 
@@ -66,7 +67,10 @@ def serialize_study_summary(summary: StudySummary) -> Dict[str, Any]:
 
 
 def serialize_study_detail(
-    summary: StudySummary, trials: List[FrozenTrial]
+    summary: StudySummary,
+    trials: List[FrozenTrial],
+    intersection: List[Tuple[str, BaseDistribution]],
+    union: List[Tuple[str, BaseDistribution]],
 ) -> Dict[str, Any]:
     serialized: Dict[str, Any] = {
         "name": summary.study_name,
@@ -83,6 +87,9 @@ def serialize_study_detail(
     serialized["trials"] = [
         serialize_frozen_trial(summary._study_id, trial) for trial in trials
     ]
+
+    serialized["intersection_search_space"] = serialize_search_space(intersection)
+    serialized["union_search_space"] = serialize_search_space(union)
     return serialized
 
 
@@ -108,3 +115,18 @@ def serialize_frozen_trial(study_id: int, trial: FrozenTrial) -> Dict[str, Any]:
         serialized["datetime_complete"] = trial.datetime_complete.isoformat()
 
     return serialized
+
+
+def serialize_search_space(
+    search_space: List[Tuple[str, BaseDistribution]]
+) -> List[Dict[str, Any]]:
+    serialized = []
+    for param_name, distribution in search_space:
+        serialized.append(
+            {
+                "name": param_name,
+                "type": distribution.__class__.__name__,
+                "attributes": distribution._asdict(),
+            }
+        )
+    return serialized

optuna_dashboard/static/apiClient.ts

@@ -168,3 +168,21 @@ export const deleteStudyAPI = (studyId: number) => {
     return {}
   })
 }
+
+interface ParamImportancesResponse {
+  target_name: string
+  param_importances: ParamImportance[]
+}
+
+export const getParamImportances = (
+  studyId: number
+): Promise<ParamImportances> => {
+  return axiosInstance
+    .get<ParamImportancesResponse>(
+      `/api/studies/${studyId}/param_importances`,
+      {}
+    )
+    .then((res) => {
+      return res.data
+    })
+}

optuna_dashboard/static/components/GraphParetoFront.tsx

@@ -0,0 +1,175 @@
+import * as plotly from "plotly.js-dist"
+import React, { FC, useEffect, useState } from "react"
+import {
+  Grid,
+  FormControl,
+  FormLabel,
+  MenuItem,
+  Select,
+} from "@material-ui/core"
+import { createStyles, makeStyles, Theme } from "@material-ui/core/styles"
+
+const plotDomId = "graph-pareto-front"
+
+const useStyles = makeStyles((theme: Theme) =>
+  createStyles({
+    formControl: {
+      marginBottom: theme.spacing(2),
+      marginRight: theme.spacing(5),
+      marginTop: theme.spacing(10),
+    },
+  })
+)
+
+export const GraphParetoFront: FC<{
+  study: StudyDetail | null
+}> = ({ study = null }) => {
+  const classes = useStyles()
+  const [objectiveXId, setObjectiveXId] = useState<number>(0)
+  const [objectiveYId, setObjectiveYId] = useState<number>(1)
+
+  const handleObjectiveXChange = (
+    event: React.ChangeEvent<{ value: unknown }>
+  ) => {
+    setObjectiveXId(event.target.value as number)
+  }
+
+  const handleObjectiveYChange = (
+    event: React.ChangeEvent<{ value: unknown }>
+  ) => {
+    setObjectiveYId(event.target.value as number)
+  }
+
+  useEffect(() => {
+    if (study != null) {
+      plotParetoFront(study, objectiveXId, objectiveYId)
+    }
+  }, [study, objectiveXId, objectiveYId])
+
+  return (
+    <Grid container direction="row">
+      {study !== null && study.directions.length !== 1 ? (
+        <Grid item xs={3}>
+          <Grid container direction="column">
+            <FormControl component="fieldset" className={classes.formControl}>
+              <FormLabel component="legend">Objective X ID:</FormLabel>
+              <Select value={objectiveXId} onChange={handleObjectiveXChange}>
+                {study.directions.map((d, i) => (
+                  <MenuItem value={i} key={i}>
+                    {i}
+                  </MenuItem>
+                ))}
+              </Select>
+            </FormControl>
+            <FormControl component="fieldset" className={classes.formControl}>
+              <FormLabel component="legend">Objective Y ID:</FormLabel>
+              <Select value={objectiveYId} onChange={handleObjectiveYChange}>
+                {study.directions.map((d, i) => (
+                  <MenuItem value={i} key={i}>
+                    {i}
+                  </MenuItem>
+                ))}
+              </Select>
+            </FormControl>
+          </Grid>
+        </Grid>
+      ) : null}
+      <Grid item xs={6}>
+        <div id={plotDomId} />
+      </Grid>
+    </Grid>
+  )
+}
+
+const plotParetoFront = (
+  study: StudyDetail,
+  objectiveXId: number,
+  objectiveYId: number
+) => {
+  if (document.getElementById(plotDomId) === null) {
+    return
+  }
+
+  const dim: number = study.directions.length
+  if (dim != 2) {
+    return
+  }
+  const layout: Partial<plotly.Layout> = {
+    title: "Pareto-front plot",
+    margin: {
+      l: 50,
+      r: 50,
+      b: 0,
+    },
+  }
+
+  const trials: Trial[] = study !== null ? study.trials : []
+  const completedTrials = trials.filter((t) => t.state === "Complete")
+
+  if (completedTrials.length === 0) {
+    plotly.react(plotDomId, [], layout)
+    return
+  }
+
+  const normalizedValues: number[][] = []
+  completedTrials.forEach((t) => {
+    if (t.values && t.values.length == dim) {
+      const trialValues = t.values.map((v: number, i: number) => {
+        return study.directions[i] === "minimize" ? v : -v
+      })
+      normalizedValues.push(trialValues)
+    }
+  })
+
+  const pointColors: string[] = []
+  normalizedValues.forEach((values0: number[], i: number) => {
+    let dominated = false
+
+    dominated = normalizedValues.some((values1: number[], j: number) => {
+      if (i === j) {
+        return false
+      }
+      return values0.every((value0: number, k: number) => {
+        return values1[k] <= value0
+      })
+    })
+
+    if (dominated) {
+      pointColors.push("blue")
+    } else {
+      pointColors.push("red")
+    }
+  })
+
+  const plotData: Partial<plotly.PlotData>[] = [
+    {
+      type: "scatter",
+      x: completedTrials.map((t: Trial): number => {
+        return t.values![objectiveXId]
+      }),
+      y: completedTrials.map((t: Trial): number => {
+        return t.values![objectiveYId]
+      }),
+      mode: "markers",
+      xaxis: "Objective X",
+      yaxis: "Objective Y",
+      marker: {
+        color: pointColors,
+      },
+      text: completedTrials.map((t: Trial): string => {
+        return JSON.stringify(
+          {
+            number: t.number,
+            values: t.values,
+            params: t.params,
+          },
+          null,
+          2
+        ).replaceAll("\n", "<br>")
+      }),
+      hovertemplate: "%{text}<extra></extra>",
+    },
+  ]
+
+  plotly.react(plotDomId, plotData, layout)
+}

optuna_dashboard/static/components/HyperparameterImportances.tsx

@@ -0,0 +1,88 @@
+import * as plotly from "plotly.js-dist"
+import React, { FC, useEffect } from "react"
+import { getParamImportances } from "../apiClient"
+const plotDomId = "graph-hyperparameter-importances"
+
+// To match colors used by plot_param_importances in optuna.
+const plotlyColorsSequentialBlues = [
+  "rgb(247,251,255)",
+  "rgb(222,235,247)",
+  "rgb(198,219,239)",
+  "rgb(158,202,225)",
+  "rgb(107,174,214)",
+  "rgb(66,146,198)",
+  "rgb(33,113,181)",
+  "rgb(8,81,156)",
+  "rgb(8,48,107)",
+]
+
+const distributionColors = {
+  UniformDistribution: plotlyColorsSequentialBlues.slice(-1)[0],
+  LogUniformDistribution: plotlyColorsSequentialBlues.slice(-1)[0],
+  DiscreteUniformDistribution: plotlyColorsSequentialBlues.slice(-1)[0],
+  IntUniformDistribution: plotlyColorsSequentialBlues.slice(-2)[0],
+  IntLogUniformDistribution: plotlyColorsSequentialBlues.slice(-2)[0],
+  CategoricalDistribution: plotlyColorsSequentialBlues.slice(-4)[0],
+}
+
+export const HyperparameterImportances: FC<{
+  studyId: number
+  numOfTrials: number
+}> = ({ studyId, numOfTrials = 0 }) => {
+  useEffect(() => {
+    async function fetchAndPlotParamImportances(studyId: number) {
+      const paramsImportanceData = await getParamImportances(studyId)
+      plotParamImportances(paramsImportanceData)
+    }
+    fetchAndPlotParamImportances(studyId)
+  }, [numOfTrials])
+  return <div id={plotDomId} />
+}
+
+const plotParamImportances = (paramsImportanceData: ParamImportances) => {
+  if (document.getElementById(plotDomId) === null) {
+    return
+  }
+  const param_importances = paramsImportanceData.param_importances.reverse()
+  const importance_values = param_importances.map((p) => p.importance)
+  const param_names = param_importances.map((p) => p.name)
+  const param_colors = param_importances.map(
+    (p) => distributionColors[p.distribution]
+  )
+  const param_hover_templates = param_importances.map(
+    (p) => `${p.name} (${p.distribution}): ${p.importance} <extra></extra>`
+  )
+
+  const layout: Partial<plotly.Layout> = {
+    title: "Hyperparameter Importance",
+    xaxis: {
+      title: `Importance for ${paramsImportanceData.target_name}`,
+    },
+    yaxis: {
+      title: "Hyperparameter",
+    },
+    margin: {
+      l: 50,
+      r: 50,
+      b: 50,
+    },
+    showlegend: false,
+  }
+
+  const plotData: Partial<plotly.PlotData>[] = [
+    {
+      type: "bar",
+      orientation: "h",
+      x: importance_values,
+      y: param_names,
+      text: importance_values.map((v) => String(v.toFixed(2))),
+      textposition: "outside",
+      hovertemplate: param_hover_templates,
+      marker: {
+        color: param_colors,
+      },
+    },
+  ]
+
+  plotly.react(plotDomId, plotData, layout)
+}

optuna_dashboard/static/components/StudyDetail.tsx

@@ -20,10 +20,12 @@ import { Home, Cached } from "@material-ui/icons"
 
 import { DataGridColumn, DataGrid } from "./DataGrid"
 import { GraphParallelCoordinate } from "./GraphParallelCoordinate"
+import { HyperparameterImportances } from "./HyperparameterImportances"
 import { GraphIntermediateValues } from "./GraphIntermediateValues"
 import { GraphSlice } from "./GraphSlice"
 import { GraphHistory } from "./GraphHistory"
 import { GraphContour } from "./GraphContour"
+import { GraphParetoFront } from "./GraphParetoFront"
 import { actionCreator } from "../action"
 import { studyDetailsState } from "../state"
 
@@ -197,6 +199,20 @@ export const StudyDetail: FC = () => {
               </Grid>
             </Grid>
           ) : null}
+          {studyDetail !== null && isSingleObjectiveStudy(studyDetail) ? (
+            <Grid container direction="row">
+              <Grid item xs={6}>
+                <Card className={classes.card}>
+                  <CardContent>
+                    <HyperparameterImportances
+                      studyId={studyIdNumber}
+                      numOfTrials={trials.length}
+                    />
+                  </CardContent>
+                </Card>
+              </Grid>
+            </Grid>
+          ) : null}
           {studyDetail !== null ? (
             <Card className={classes.card}>
               <CardContent>
@@ -210,6 +226,13 @@ export const StudyDetail: FC = () => {
                 <GraphContour study={studyDetail} />
               </CardContent>
             </Card>
+          ): null}
+          {studyDetail !== null && !isSingleObjectiveStudy(studyDetail) ? (
+            <Card className={classes.card}>
+              <CardContent>
+                <GraphParetoFront study={studyDetail} />
+              </CardContent>
+            </Card>
           ) : null}
           <Card className={classes.card}>
             <TrialTable studyDetail={studyDetail} />

optuna_dashboard/static/types/index.d.ts

@@ -9,6 +9,13 @@ declare const URL_PREFIX: string
 
 type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting"
 type StudyDirection = "maximize" | "minimize" | "not_set"
+type Distribution =
+  | "UniformDistribution"
+  | "LogUniformDistribution"
+  | "DiscreteUniformDistribution"
+  | "IntUniformDistribution"
+  | "IntLogUniformDistribution"
+  | "CategoricalDistribution"
 
 declare interface TrialIntermediateValue {
   step: number
@@ -20,6 +27,12 @@ declare interface TrialParam {
   value: string
 }
 
+declare interface ParamImportance {
+  name: string
+  importance: number
+  distribution: Distribution
+}
+
 declare interface Attribute {
   key: string
   value: string
@@ -60,3 +73,8 @@ declare interface StudyDetail {
 declare interface StudyDetails {
   [study_id: string]: StudyDetail
 }
+
+declare interface ParamImportances {
+  target_name: string
+  param_importances: ParamImportance[]
+}

setup.cfg

@@ -30,6 +30,7 @@ install_requires =
     optuna>=2.4
     bottle
     typing-extensions;python_version<'3.8'
+    scikit-learn
 
 [options.extras_require]
 lint =

tests/test_search_space.py

@@ -0,0 +1,115 @@
+import warnings
+from unittest import TestCase
+
+import optuna
+from optuna import create_trial
+from optuna.distributions import UniformDistribution
+from optuna.exceptions import ExperimentalWarning
+from optuna.trial import TrialState
+
+from optuna_dashboard.search_space import _SearchSpace
+
+
+class SearchSpaceTestCase(TestCase):
+    def setUp(self) -> None:
+        optuna.logging.set_verbosity(optuna.logging.ERROR)
+        warnings.simplefilter("ignore", category=ExperimentalWarning)
+
+    def test_same_distributions(self) -> None:
+        distributions = [
+            {
+                "x0": UniformDistribution(low=0, high=10),
+                "x1": UniformDistribution(low=0, high=10),
+            },
+            {
+                "x0": UniformDistribution(low=0, high=10),
+                "x1": UniformDistribution(low=0, high=10),
+            },
+        ]
+        params = [
+            {
+                "x0": 0.5,
+                "x1": 0.5,
+            },
+            {
+                "x0": 0.5,
+                "x1": 0.5,
+            },
+        ]
+        trials = [
+            create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p)
+            for d, p in zip(distributions, params)
+        ]
+        search_space = _SearchSpace()
+        search_space.update(trials)
+
+        self.assertEqual(len(search_space.intersection), 2)
+        self.assertEqual(len(search_space.union), 2)
+
+    def test_different_distributions(self) -> None:
+        distributions = [
+            {
+                "x0": UniformDistribution(low=0, high=10),
+                "x1": UniformDistribution(low=0, high=10),
+            },
+            {
+                "x0": UniformDistribution(low=0, high=5),
+                "x1": UniformDistribution(low=0, high=10),
+            },
+        ]
+        params = [
+            {
+                "x0": 0.5,
+                "x1": 0.5,
+            },
+            {
+                "x0": 0.5,
+                "x1": 0.5,
+            },
+        ]
+        trials = [
+            create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p)
+            for d, p in zip(distributions, params)
+        ]
+        search_space = _SearchSpace()
+        search_space.update(trials)
+
+        self.assertEqual(len(search_space.intersection), 1)
+        self.assertEqual(len(search_space.union), 3)
+
+    def test_dynamic_search_space(self) -> None:
+        distributions = [
+            {
+                "x0": UniformDistribution(low=0, high=10),
+                "x1": UniformDistribution(low=0, high=10),
+            },
+            {
+                "x0": UniformDistribution(low=0, high=5),
+            },
+            {
+                "x0": UniformDistribution(low=0, high=10),
+                "x1": UniformDistribution(low=0, high=10),
+            },
+        ]
+        params = [
+            {
+                "x0": 0.5,
+                "x1": 0.5,
+            },
+            {
+                "x0": 0.5,
+            },
+            {
+                "x0": 0.5,
+                "x1": 0.5,
+            },
+        ]
+        trials = [
+            create_trial(state=TrialState.COMPLETE, value=0, distributions=d, params=p)
+            for d, p in zip(distributions, params)
+        ]
+        search_space = _SearchSpace()
+        search_space.update(trials)
+
+        self.assertEqual(len(search_space.intersection), 0)
+        self.assertEqual(len(search_space.union), 3)