Merge pull request #79 from optuna/revert-graph-contour

Revert graph contour because it will be crashed on dynamic search space.

optuna_dashboard/static/components/GraphContour.tsx

@@ -1,278 +0,0 @@
-import * as plotly from "plotly.js-dist"
-import React, { ChangeEvent, FC, useEffect, useState } from "react"
-import {
-  Grid,
-  FormControl,
-  FormLabel,
-  InputLabel,
-  MenuItem,
-  Select,
-  Typography,
-} from "@material-ui/core"
-import { createStyles, makeStyles, Theme } from "@material-ui/core/styles"
-
-const plotDomId = "graph-contour"
-
-const useStyles = makeStyles((theme: Theme) =>
-  createStyles({
-    title: {
-      margin: "1em 0",
-    },
-    formControl: {
-      marginBottom: theme.spacing(2),
-      marginRight: theme.spacing(2),
-    },
-  })
-)
-
-const getParamNames = (trials: Trial[]): string[] => {
-  const paramSet = new Set<string>(
-    ...trials.map<string[]>((t) => t.params.map((p) => p.name))
-  )
-  return Array.from(paramSet)
-}
-
-export const GraphContour: FC<{
-  study: StudyDetail | null
-}> = ({ study = null }) => {
-  const trials: Trial[] = study !== null ? study.trials : []
-  const classes = useStyles()
-  const [paramNames, setParamNames] = useState<string[]>([])
-  const [objectiveId, setObjectiveId] = useState<number>(0)
-  const [xAxis, setXAxis] = useState<string | null>(null)
-  const [yAxis, setYAxis] = useState<string | null>(null)
-
-  useEffect(() => {
-    if (trials.length === 0 || paramNames.length !== 0) {
-      return
-    }
-    const filteredTrials = trials.filter(
-      (t) =>
-        t.state === "Complete" ||
-        (t.state === "Pruned" && t.values && t.values.length > 0)
-    )
-
-    const p = getParamNames(filteredTrials)
-    if (p.length === 0 || p.length === paramNames.length) {
-      return
-    }
-    setParamNames(p)
-    if (p.length < 2 && xAxis !== null && yAxis !== null) {
-      return
-    }
-    setXAxis(p[0])
-    setYAxis(p[1])
-  }, [trials])
-
-  const handleObjectiveChange = (
-    event: React.ChangeEvent<{ value: unknown }>
-  ) => {
-    setObjectiveId(event.target.value as number)
-  }
-
-  const handleXAxisChange = (e: ChangeEvent<{ value: unknown }>) => {
-    setXAxis(e.target.value as string)
-  }
-
-  const handleYAxisChange = (e: ChangeEvent<{ value: unknown }>) => {
-    setYAxis(e.target.value as string)
-  }
-
-  useEffect(() => {
-    if (study !== null) {
-      plotContour(study, objectiveId, xAxis, yAxis, paramNames)
-    }
-  }, [study, objectiveId, xAxis, yAxis, paramNames])
-
-  return (
-    <Grid container direction="row">
-      <Grid item xs={3}>
-        <Grid container direction="column">
-          <Typography variant="h6" className={classes.title}>
-            Contour
-          </Typography>
-          {study !== null && study.directions.length !== 1 ? (
-            <FormControl component="fieldset" className={classes.formControl}>
-              <FormLabel component="legend">Objective ID:</FormLabel>
-              <Select value={objectiveId} onChange={handleObjectiveChange}>
-                {study.directions.map((d, i) => (
-                  <MenuItem value={i} key={i}>
-                    {i}
-                  </MenuItem>
-                ))}
-              </Select>
-            </FormControl>
-          ) : null}
-          <FormControl component="fieldset" className={classes.formControl}>
-            <InputLabel id="parameter1">X Axis Parameter</InputLabel>
-            <Select value={xAxis || ""} onChange={handleXAxisChange}>
-              {paramNames.map((x) => (
-                <MenuItem value={x} key={x}>
-                  {x}
-                </MenuItem>
-              ))}
-            </Select>
-          </FormControl>
-          <FormControl component="fieldset" className={classes.formControl}>
-            <InputLabel id="parameter2">Y Axis Parameter</InputLabel>
-            <Select value={yAxis || ""} onChange={handleYAxisChange}>
-              {paramNames.map((x) => (
-                <MenuItem value={x} key={x}>
-                  {x}
-                </MenuItem>
-              ))}
-            </Select>
-          </FormControl>
-        </Grid>
-      </Grid>
-      <Grid item xs={9}>
-        <div id={plotDomId} />
-      </Grid>
-    </Grid>
-  )
-}
-
-const plotContour = (
-  study: StudyDetail,
-  objectiveId: number,
-  xAxis: string | null,
-  yAxis: string | null,
-  paramNames: string[]
-) => {
-  if (document.getElementById(plotDomId) === null) {
-    return
-  }
-
-  const layout: Partial<plotly.Layout> = {
-    margin: {
-      l: 50,
-      t: 0,
-      r: 50,
-      b: 0,
-    },
-    xaxis: {
-      gridcolor: "#f2f5fa",
-      gridwidth: 1,
-      zerolinecolor: "#f2f5fa",
-      zerolinewidth: 1.5,
-    },
-    yaxis: {
-      gridcolor: "#f2f5fa",
-      gridwidth: 1,
-      zerolinecolor: "#f2f5fa",
-      zerolinewidth: 1.5,
-    },
-    plot_bgcolor: "#E5ecf6",
-  }
-
-  const trials: Trial[] = study !== null ? study.trials : []
-
-  const filteredTrials = trials.filter(
-    (t) =>
-      t.state === "Complete" ||
-      (t.state === "Pruned" && t.values && t.values.length > 0)
-  )
-
-  if (filteredTrials.length === 0 || xAxis === null || yAxis === null) {
-    plotly.react(plotDomId, [], layout)
-  }
-
-  if (paramNames.length === 0 || paramNames.length === 1) {
-    plotly.react(plotDomId, [], layout)
-    return
-  }
-  if (xAxis === yAxis) {
-    plotly.react(plotDomId, [], layout)
-    return
-  }
-
-  const objectiveValues: number[] = filteredTrials.map(
-    (t) => t.values![objectiveId]
-  )
-
-  const paramValues: { [key: number]: string[] } = []
-  const paramIndices: Array<string> = []
-
-  if (paramNames.length >= 2) {
-    paramNames.forEach((paramName, index) => {
-      const valueStrings = filteredTrials.map((t) => {
-        const param = t.params.find((p) => p.name == paramName)
-        return param!.value
-      })
-      paramValues[index] = valueStrings
-      paramIndices[index] = paramName
-    })
-
-    const paramCategorical = { ...paramValues }
-    let xIndex = 0
-    let yIndex = 0
-    if (typeof xAxis === "string" && typeof yAxis === "string") {
-      xIndex = paramIndices.indexOf(xAxis)
-      yIndex = paramIndices.indexOf(yAxis)
-    }
-
-    const xIndice = paramCategorical[xIndex].sort((a, b) => (a > b ? 1 : -1))
-    const yIndice = paramCategorical[yIndex].sort((a, b) => (a > b ? 1 : -1))
-
-    const xIndices: string[] = []
-    const yIndices: string[] = []
-
-    xIndice.forEach((element) => {
-      if (!xIndices.includes(element)) {
-        xIndices.push(element)
-      }
-    })
-
-    yIndice.forEach((element) => {
-      if (!yIndices.includes(element)) {
-        yIndices.push(element)
-      }
-    })
-
-    const z: number[][] = []
-    for (let j = 0; j < yIndices.length; j++) {
-      z[j] = []
-    }
-    for (let j = 0; j < filteredTrials.length; j++) {
-      const xI = xIndices.indexOf(paramValues[xIndex][j])
-      const yI = yIndices.indexOf(paramValues[yIndex][j])
-      z[yI][xI] = objectiveValues[j]
-    }
-
-    const data: Partial<plotly.PlotData>[] = [
-      {
-        type: "contour",
-        z: z,
-        x: xIndices,
-        y: yIndices,
-        mode: "markers",
-        marker: {
-          color: "#000",
-        },
-        line: {
-          color: "#000",
-        },
-        //@ts-ignore
-        colorbar: {
-          title: "Objective Value",
-        },
-        colorscale: "Blues",
-        connectgaps: true,
-        contours_coloring: "heatmap",
-        hoverinfo: "none",
-        line_smoothing: 1.3,
-      },
-      {
-        type: "scatter",
-        x: paramValues[xIndex],
-        y: paramValues[yIndex],
-        mode: "markers",
-        marker: {
-          color: "#000",
-        },
-      },
-    ]
-
-    plotly.react(plotDomId, data, layout)
-  }
-}

optuna_dashboard/static/components/StudyDetail.tsx

@@ -26,7 +26,6 @@ import { Edf } from "./GraphEdf"
 import { GraphIntermediateValues } from "./GraphIntermediateValues"
 import { GraphSlice } from "./GraphSlice"
 import { GraphHistory } from "./GraphHistory"
-import { GraphContour } from "./GraphContour"
 import { GraphParetoFront } from "./GraphParetoFront"
 import { actionCreator } from "../action"
 import { studyDetailsState } from "../state"
@@ -236,13 +235,6 @@ export const StudyDetail: FC = () => {
               </CardContent>
             </Card>
           ) : null}
-          {studyDetail !== null ? (
-            <Card className={classes.card}>
-              <CardContent>
-                <GraphContour study={studyDetail} />
-              </CardContent>
-            </Card>
-          ) : null}
           <Card className={classes.card}>
             <TrialTable studyDetail={studyDetail} />
           </Card>

visual_regression_test.py

@@ -1,8 +1,8 @@
+import argparse
 import asyncio
 import os
 import threading
 import time
-from typing import List
 from typing import Tuple
 from wsgiref.simple_server import make_server
 
@@ -12,16 +12,30 @@ from pyppeteer import launch
 from optuna_dashboard.app import create_app
 
 
-host = "127.0.0.1"
-port = 8080
-output_dir = "tmp"
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--port", help="port number (default: %(default)s)", type=int, default=8081
+)
+parser.add_argument(
+    "--host", help="hostname (default: %(default)s)", default="127.0.0.1"
+)
+parser.add_argument(
+    "--sleep",
+    help="sleep seconds on each page open (default: %(default)s)",
+    type=int,
+    default="10",
+)
+parser.add_argument(
+    "--output-dir", help="output directory (default: %(default)s)", default="tmp"
+)
+args = parser.parse_args()
 
 
 def create_optuna_storage() -> optuna.storages.InMemoryStorage:
     storage = optuna.storages.InMemoryStorage()
 
     # Single-objective study
-    study = optuna.create_study(study_name="single-objective", storage=storage)
+    study = optuna.create_study(study_name="single", storage=storage)
 
     def objective_single(trial: optuna.Trial) -> float:
         x1 = trial.suggest_float("x1", 0, 10)
@@ -32,7 +46,7 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage:
 
     # Single-objective study with 1 parameter
     study = optuna.create_study(
-        study_name="single-objective-1-param", storage=storage, direction="maximize"
+        study_name="single-1-param", storage=storage, direction="maximize"
     )
 
     def objective_single_with_1param(trial: optuna.Trial) -> float:
@@ -41,6 +55,20 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage:
 
     study.optimize(objective_single_with_1param, n_trials=50)
 
+    # Single-objective study with dynamic search space
+    study = optuna.create_study(
+        study_name="single-dynamic", storage=storage, direction="maximize"
+    )
+
+    def objective_single_dynamic(trial: optuna.Trial) -> float:
+        category = trial.suggest_categorical("category", ["foo", "bar"])
+        if category == "foo":
+            return (trial.suggest_float("x1", 0, 10) - 2) ** 2
+        else:
+            return -((trial.suggest_float("x2", -10, 0) + 5) ** 2)
+
+    study.optimize(objective_single_dynamic, n_trials=50)
+
     # Multi-objective study
     study = optuna.create_study(
         study_name="multi-objective",
@@ -57,12 +85,34 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage:
 
     study.optimize(objective_multi, n_trials=50)
 
-    # Pruning with no intermediate values
+    # Multi-objective study with dynamic search space
     study = optuna.create_study(
-        study_name="binh-korn-function-with-constraints", storage=storage
+        study_name="multi-dynamic", storage=storage, directions=["minimize", "minimize"]
     )
 
-    def objective_prune_with_no_trials(trial: optuna.Trial) -> float:
+    def objective_multi_dynamic(trial: optuna.Trial) -> Tuple[float, float]:
+        category = trial.suggest_categorical("category", ["foo", "bar"])
+        if category == "foo":
+            x = trial.suggest_float("x1", 0, 5)
+            y = trial.suggest_float("y1", 0, 3)
+            v0 = 4 * x ** 2 + 4 * y ** 2
+            v1 = (x - 5) ** 2 + (y - 5) ** 2
+            return v0, v1
+        else:
+            x = trial.suggest_float("x2", 0, 5)
+            y = trial.suggest_float("y2", 0, 3)
+            v0 = 2 * x ** 2 + 2 * y ** 2
+            v1 = (x - 2) ** 2 + (y - 3) ** 2
+            return v0, v1
+
+    study.optimize(objective_multi_dynamic, n_trials=50)
+
+    # Pruning with no intermediate values
+    study = optuna.create_study(
+        study_name="single-pruned-without-report", storage=storage
+    )
+
+    def objective_prune_without_report(trial: optuna.Trial) -> float:
         x = trial.suggest_float("x", -15, 30)
         y = trial.suggest_float("y", -15, 30)
         v = x ** 2 + y ** 2
@@ -70,52 +120,49 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage:
             raise optuna.TrialPruned()
         return v
 
-    study.optimize(objective_prune_with_no_trials, n_trials=100)
+    study.optimize(objective_prune_without_report, n_trials=100)
 
     # No trials single-objective study
-    optuna.create_study(
-        study_name="single-objective study with no trials", storage=storage
-    )
+    optuna.create_study(study_name="single-no-trials", storage=storage)
 
     # No trials multi-objective study
     optuna.create_study(
-        study_name="multi-objective study with no trials",
+        study_name="multi-no-trials",
         storage=storage,
         directions=["minimize", "maximize"],
     )
     return storage
 
 
-async def take_screenshots(study_ids: List[int]) -> None:
+async def take_screenshots(storage: optuna.storages.BaseStorage) -> None:
     browser = await launch()
     page = await browser.newPage()
     await page.setViewport({"width": 1000, "height": 3000})
 
-    await page.goto(f"http://{host}:{port}/dashboard/")
+    await page.goto(f"http://{args.host}:{args.port}/dashboard/")
     time.sleep(1)
-    await page.screenshot({"path": os.path.join(output_dir, "study-list.png")})
+    await page.screenshot({"path": os.path.join(args.output_dir, "study-list.png")})
 
-    for study_id in study_ids:
-        await page.goto(f"http://{host}:{port}/dashboard/studies/{study_id}")
-        time.sleep(10)
+    study_ids = {s._study_id: s.study_name for s in storage.get_all_study_summaries()}
+    for study_id, study_name in study_ids.items():
+        await page.goto(f"http://{args.host}:{args.port}/dashboard/studies/{study_id}")
+        time.sleep(args.sleep)
         await page.screenshot(
-            {"path": os.path.join(output_dir, f"study-{study_id}.png")}
+            {"path": os.path.join(args.output_dir, f"study-{study_name}.png")}
         )
     await browser.close()
 
 
 def main() -> None:
-    os.makedirs(output_dir, exist_ok=True)
-
+    os.makedirs(args.output_dir, exist_ok=True)
     storage = create_optuna_storage()
     app = create_app(storage)
-    httpd = make_server(host, port, app)
+    httpd = make_server(args.host, args.port, app)
     thread = threading.Thread(target=httpd.serve_forever)
     thread.start()
 
-    study_ids = [s._study_id for s in storage.get_all_study_summaries()]
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(take_screenshots(study_ids))
+    loop.run_until_complete(take_screenshots(storage))
 
     httpd.shutdown()
     httpd.server_close()