diff --git a/optuna_dashboard/static/components/GraphContour.tsx b/optuna_dashboard/static/components/GraphContour.tsx deleted file mode 100644 index 85f410c9..00000000 --- a/optuna_dashboard/static/components/GraphContour.tsx +++ /dev/null @@ -1,278 +0,0 @@ -import * as plotly from "plotly.js-dist" -import React, { ChangeEvent, FC, useEffect, useState } from "react" -import { - Grid, - FormControl, - FormLabel, - InputLabel, - MenuItem, - Select, - Typography, -} from "@material-ui/core" -import { createStyles, makeStyles, Theme } from "@material-ui/core/styles" - -const plotDomId = "graph-contour" - -const useStyles = makeStyles((theme: Theme) => - createStyles({ - title: { - margin: "1em 0", - }, - formControl: { - marginBottom: theme.spacing(2), - marginRight: theme.spacing(2), - }, - }) -) - -const getParamNames = (trials: Trial[]): string[] => { - const paramSet = new Set( - ...trials.map((t) => t.params.map((p) => p.name)) - ) - return Array.from(paramSet) -} - -export const GraphContour: FC<{ - study: StudyDetail | null -}> = ({ study = null }) => { - const trials: Trial[] = study !== null ? study.trials : [] - const classes = useStyles() - const [paramNames, setParamNames] = useState([]) - const [objectiveId, setObjectiveId] = useState(0) - const [xAxis, setXAxis] = useState(null) - const [yAxis, setYAxis] = useState(null) - - useEffect(() => { - if (trials.length === 0 || paramNames.length !== 0) { - return - } - const filteredTrials = trials.filter( - (t) => - t.state === "Complete" || - (t.state === "Pruned" && t.values && t.values.length > 0) - ) - - const p = getParamNames(filteredTrials) - if (p.length === 0 || p.length === paramNames.length) { - return - } - setParamNames(p) - if (p.length < 2 && xAxis !== null && yAxis !== null) { - return - } - setXAxis(p[0]) - setYAxis(p[1]) - }, [trials]) - - const handleObjectiveChange = ( - event: React.ChangeEvent<{ value: unknown }> - ) => { - setObjectiveId(event.target.value as number) - } - - const handleXAxisChange = (e: ChangeEvent<{ value: unknown }>) => { - setXAxis(e.target.value as string) - } - - const handleYAxisChange = (e: ChangeEvent<{ value: unknown }>) => { - setYAxis(e.target.value as string) - } - - useEffect(() => { - if (study !== null) { - plotContour(study, objectiveId, xAxis, yAxis, paramNames) - } - }, [study, objectiveId, xAxis, yAxis, paramNames]) - - return ( - - - - - Contour - - {study !== null && study.directions.length !== 1 ? ( - - Objective ID: - - - ) : null} - - X Axis Parameter - - - - Y Axis Parameter - - - - - -
- - - ) -} - -const plotContour = ( - study: StudyDetail, - objectiveId: number, - xAxis: string | null, - yAxis: string | null, - paramNames: string[] -) => { - if (document.getElementById(plotDomId) === null) { - return - } - - const layout: Partial = { - margin: { - l: 50, - t: 0, - r: 50, - b: 0, - }, - xaxis: { - gridcolor: "#f2f5fa", - gridwidth: 1, - zerolinecolor: "#f2f5fa", - zerolinewidth: 1.5, - }, - yaxis: { - gridcolor: "#f2f5fa", - gridwidth: 1, - zerolinecolor: "#f2f5fa", - zerolinewidth: 1.5, - }, - plot_bgcolor: "#E5ecf6", - } - - const trials: Trial[] = study !== null ? study.trials : [] - - const filteredTrials = trials.filter( - (t) => - t.state === "Complete" || - (t.state === "Pruned" && t.values && t.values.length > 0) - ) - - if (filteredTrials.length === 0 || xAxis === null || yAxis === null) { - plotly.react(plotDomId, [], layout) - } - - if (paramNames.length === 0 || paramNames.length === 1) { - plotly.react(plotDomId, [], layout) - return - } - if (xAxis === yAxis) { - plotly.react(plotDomId, [], layout) - return - } - - const objectiveValues: number[] = filteredTrials.map( - (t) => t.values![objectiveId] - ) - - const paramValues: { [key: number]: string[] } = [] - const paramIndices: Array = [] - - if (paramNames.length >= 2) { - paramNames.forEach((paramName, index) => { - const valueStrings = filteredTrials.map((t) => { - const param = t.params.find((p) => p.name == paramName) - return param!.value - }) - paramValues[index] = valueStrings - paramIndices[index] = paramName - }) - - const paramCategorical = { ...paramValues } - let xIndex = 0 - let yIndex = 0 - if (typeof xAxis === "string" && typeof yAxis === "string") { - xIndex = paramIndices.indexOf(xAxis) - yIndex = paramIndices.indexOf(yAxis) - } - - const xIndice = paramCategorical[xIndex].sort((a, b) => (a > b ? 1 : -1)) - const yIndice = paramCategorical[yIndex].sort((a, b) => (a > b ? 1 : -1)) - - const xIndices: string[] = [] - const yIndices: string[] = [] - - xIndice.forEach((element) => { - if (!xIndices.includes(element)) { - xIndices.push(element) - } - }) - - yIndice.forEach((element) => { - if (!yIndices.includes(element)) { - yIndices.push(element) - } - }) - - const z: number[][] = [] - for (let j = 0; j < yIndices.length; j++) { - z[j] = [] - } - for (let j = 0; j < filteredTrials.length; j++) { - const xI = xIndices.indexOf(paramValues[xIndex][j]) - const yI = yIndices.indexOf(paramValues[yIndex][j]) - z[yI][xI] = objectiveValues[j] - } - - const data: Partial[] = [ - { - type: "contour", - z: z, - x: xIndices, - y: yIndices, - mode: "markers", - marker: { - color: "#000", - }, - line: { - color: "#000", - }, - //@ts-ignore - colorbar: { - title: "Objective Value", - }, - colorscale: "Blues", - connectgaps: true, - contours_coloring: "heatmap", - hoverinfo: "none", - line_smoothing: 1.3, - }, - { - type: "scatter", - x: paramValues[xIndex], - y: paramValues[yIndex], - mode: "markers", - marker: { - color: "#000", - }, - }, - ] - - plotly.react(plotDomId, data, layout) - } -} diff --git a/optuna_dashboard/static/components/StudyDetail.tsx b/optuna_dashboard/static/components/StudyDetail.tsx index f2e9b2c7..b328a70f 100644 --- a/optuna_dashboard/static/components/StudyDetail.tsx +++ b/optuna_dashboard/static/components/StudyDetail.tsx @@ -26,7 +26,6 @@ import { Edf } from "./GraphEdf" import { GraphIntermediateValues } from "./GraphIntermediateValues" import { GraphSlice } from "./GraphSlice" import { GraphHistory } from "./GraphHistory" -import { GraphContour } from "./GraphContour" import { GraphParetoFront } from "./GraphParetoFront" import { actionCreator } from "../action" import { studyDetailsState } from "../state" @@ -236,13 +235,6 @@ export const StudyDetail: FC = () => { ) : null} - {studyDetail !== null ? ( - - - - - - ) : null} diff --git a/visual_regression_test.py b/visual_regression_test.py index e108cd5a..1a377cf7 100644 --- a/visual_regression_test.py +++ b/visual_regression_test.py @@ -1,8 +1,8 @@ +import argparse import asyncio import os import threading import time -from typing import List from typing import Tuple from wsgiref.simple_server import make_server @@ -12,16 +12,30 @@ from pyppeteer import launch from optuna_dashboard.app import create_app -host = "127.0.0.1" -port = 8080 -output_dir = "tmp" +parser = argparse.ArgumentParser() +parser.add_argument( + "--port", help="port number (default: %(default)s)", type=int, default=8081 +) +parser.add_argument( + "--host", help="hostname (default: %(default)s)", default="127.0.0.1" +) +parser.add_argument( + "--sleep", + help="sleep seconds on each page open (default: %(default)s)", + type=int, + default="10", +) +parser.add_argument( + "--output-dir", help="output directory (default: %(default)s)", default="tmp" +) +args = parser.parse_args() def create_optuna_storage() -> optuna.storages.InMemoryStorage: storage = optuna.storages.InMemoryStorage() # Single-objective study - study = optuna.create_study(study_name="single-objective", storage=storage) + study = optuna.create_study(study_name="single", storage=storage) def objective_single(trial: optuna.Trial) -> float: x1 = trial.suggest_float("x1", 0, 10) @@ -32,7 +46,7 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage: # Single-objective study with 1 parameter study = optuna.create_study( - study_name="single-objective-1-param", storage=storage, direction="maximize" + study_name="single-1-param", storage=storage, direction="maximize" ) def objective_single_with_1param(trial: optuna.Trial) -> float: @@ -41,6 +55,20 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage: study.optimize(objective_single_with_1param, n_trials=50) + # Single-objective study with dynamic search space + study = optuna.create_study( + study_name="single-dynamic", storage=storage, direction="maximize" + ) + + def objective_single_dynamic(trial: optuna.Trial) -> float: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + return (trial.suggest_float("x1", 0, 10) - 2) ** 2 + else: + return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) + + study.optimize(objective_single_dynamic, n_trials=50) + # Multi-objective study study = optuna.create_study( study_name="multi-objective", @@ -57,12 +85,34 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage: study.optimize(objective_multi, n_trials=50) - # Pruning with no intermediate values + # Multi-objective study with dynamic search space study = optuna.create_study( - study_name="binh-korn-function-with-constraints", storage=storage + study_name="multi-dynamic", storage=storage, directions=["minimize", "minimize"] ) - def objective_prune_with_no_trials(trial: optuna.Trial) -> float: + def objective_multi_dynamic(trial: optuna.Trial) -> Tuple[float, float]: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + x = trial.suggest_float("x1", 0, 5) + y = trial.suggest_float("y1", 0, 3) + v0 = 4 * x ** 2 + 4 * y ** 2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + else: + x = trial.suggest_float("x2", 0, 5) + y = trial.suggest_float("y2", 0, 3) + v0 = 2 * x ** 2 + 2 * y ** 2 + v1 = (x - 2) ** 2 + (y - 3) ** 2 + return v0, v1 + + study.optimize(objective_multi_dynamic, n_trials=50) + + # Pruning with no intermediate values + study = optuna.create_study( + study_name="single-pruned-without-report", storage=storage + ) + + def objective_prune_without_report(trial: optuna.Trial) -> float: x = trial.suggest_float("x", -15, 30) y = trial.suggest_float("y", -15, 30) v = x ** 2 + y ** 2 @@ -70,52 +120,49 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage: raise optuna.TrialPruned() return v - study.optimize(objective_prune_with_no_trials, n_trials=100) + study.optimize(objective_prune_without_report, n_trials=100) # No trials single-objective study - optuna.create_study( - study_name="single-objective study with no trials", storage=storage - ) + optuna.create_study(study_name="single-no-trials", storage=storage) # No trials multi-objective study optuna.create_study( - study_name="multi-objective study with no trials", + study_name="multi-no-trials", storage=storage, directions=["minimize", "maximize"], ) return storage -async def take_screenshots(study_ids: List[int]) -> None: +async def take_screenshots(storage: optuna.storages.BaseStorage) -> None: browser = await launch() page = await browser.newPage() await page.setViewport({"width": 1000, "height": 3000}) - await page.goto(f"http://{host}:{port}/dashboard/") + await page.goto(f"http://{args.host}:{args.port}/dashboard/") time.sleep(1) - await page.screenshot({"path": os.path.join(output_dir, "study-list.png")}) + await page.screenshot({"path": os.path.join(args.output_dir, "study-list.png")}) - for study_id in study_ids: - await page.goto(f"http://{host}:{port}/dashboard/studies/{study_id}") - time.sleep(10) + study_ids = {s._study_id: s.study_name for s in storage.get_all_study_summaries()} + for study_id, study_name in study_ids.items(): + await page.goto(f"http://{args.host}:{args.port}/dashboard/studies/{study_id}") + time.sleep(args.sleep) await page.screenshot( - {"path": os.path.join(output_dir, f"study-{study_id}.png")} + {"path": os.path.join(args.output_dir, f"study-{study_name}.png")} ) await browser.close() def main() -> None: - os.makedirs(output_dir, exist_ok=True) - + os.makedirs(args.output_dir, exist_ok=True) storage = create_optuna_storage() app = create_app(storage) - httpd = make_server(host, port, app) + httpd = make_server(args.host, args.port, app) thread = threading.Thread(target=httpd.serve_forever) thread.start() - study_ids = [s._study_id for s in storage.get_all_study_summaries()] loop = asyncio.get_event_loop() - loop.run_until_complete(take_screenshots(study_ids)) + loop.run_until_complete(take_screenshots(storage)) httpd.shutdown() httpd.server_close()