From c0bb2486eb2652c16e892c38656bd1616a2a2471 Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 6 Jan 2023 18:38:45 +0900 Subject: [PATCH 01/14] Refactor history component --- .../ts/components/GraphHistory.tsx | 150 ++++-------------- optuna_dashboard/ts/trialFilter.ts | 112 +++++++++++++ 2 files changed, 139 insertions(+), 123 deletions(-) create mode 100644 optuna_dashboard/ts/trialFilter.ts diff --git a/optuna_dashboard/ts/components/GraphHistory.tsx b/optuna_dashboard/ts/components/GraphHistory.tsx index b7c13940..4f55d735 100644 --- a/optuna_dashboard/ts/components/GraphHistory.tsx +++ b/optuna_dashboard/ts/components/GraphHistory.tsx @@ -1,5 +1,5 @@ import * as plotly from "plotly.js-dist-min" -import React, { ChangeEvent, FC, useEffect, useMemo, useState } from "react" +import React, { ChangeEvent, FC, useEffect, useState } from "react" import { Grid, FormControl, @@ -16,120 +16,43 @@ import { useTheme, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" +import { useFilteredTrials, Target, useTargetList } from "../trialFilter" const plotDomId = "graph-history" -class Target { - kind: "objective" | "user_attr" - key: number | string - - constructor(kind: "objective" | "user_attr", key: number | string) { - this.kind = kind - this.key = key - } - - validate(): boolean { - if (this.kind === "objective") { - if (typeof this.key !== "number") { - return false - } - } else if (this.kind === "user_attr") { - if (typeof this.key !== "string") { - return false - } - } else { - return false - } - return true - } - - toLabel(objectiveNames: string[]): string { - if (this.kind === "objective") { - const objectiveId: number = this.key as number - if (objectiveNames.length > objectiveId) { - return objectiveNames[objectiveId] - } - return `Objective ${objectiveId}` - } else { - return `User Attribute ${this.key}` - } - } - - getObjectiveId(): number | null { - return this.key as number - } - - getTargetValue(trial: Trial): number | null { - if (!this.validate()) { - return null - } - if (this.kind === "objective") { - const objectiveId = this.getObjectiveId() - if ( - objectiveId === null || - trial.values === undefined || - trial.values.length <= objectiveId - ) { - return null - } - const value = trial.values[objectiveId] - if (value === "inf" || value === "-inf") { - return null - } - return value - } else if (this.kind === "user_attr") { - const attr = trial.user_attrs.find((attr) => attr.key === this.key) - if (attr === undefined) { - return null - } - const value = Number(attr.value) - if (value === undefined) { - return null - } - return value - } - return null - } -} - export const GraphHistory: FC<{ study: StudyDetail | null }> = ({ study = null }) => { const theme = useTheme() const [xAxis, setXAxis] = useState("number") - const [targetIndex, setTargetIndex] = useState(0) const [logScale, setLogScale] = useState(false) const [filterCompleteTrial, setFilterCompleteTrial] = useState(false) const [filterPrunedTrial, setFilterPrunedTrial] = useState(false) - const [targetList, setTargetList] = useState([]) - const objectiveNames: string[] = study?.objective_names || [] - useMemo(() => { - if (study !== null) { - const targets: Target[] = [ - ...study.directions.map((v, i) => new Target("objective", i)), - ...study.union_user_attrs - .filter((attr) => attr.sortable) - .map((attr) => new Target("user_attr", attr.key)), - ] - setTargetList(targets) - } - }, [study?.directions, study?.union_user_attrs]) + const objectiveNames: string[] = study?.objective_names || [] + const targetList = useTargetList(study) + const [targetIndex, setTargetIndex] = useState(0) + const trials = useFilteredTrials( + study, + targetList[targetIndex], + filterCompleteTrial, + filterPrunedTrial + ) useEffect(() => { if (study !== null) { plotHistory( - study, + trials, + study.directions, targetList[targetIndex], xAxis, logScale, - filterCompleteTrial, - filterPrunedTrial, theme.palette.mode ) } }, [ - study, + trials, + study?.directions, targetIndex, targetList, logScale, @@ -258,21 +181,12 @@ export const GraphHistory: FC<{ ) } -const filterFunc = (trial: Trial, target: Target): boolean => { - if (trial.state !== "Complete" && trial.state !== "Pruned") { - return false - } - const value = target.getTargetValue(trial) - return value !== null -} - const plotHistory = ( - study: StudyDetail, + trials: Trial[], + directions: StudyDirection[], target: Target, xAxis: string, logScale: boolean, - filterCompleteTrial: boolean, - filterPrunedTrial: boolean, mode: string ) => { if (document.getElementById(plotDomId) === null) { @@ -297,15 +211,7 @@ const plotHistory = ( showlegend: true, template: mode === "dark" ? plotlyDarkTemplate : {}, } - - let filteredTrials = study.trials.filter((t) => filterFunc(t, target)) - if (filterCompleteTrial) { - filteredTrials = filteredTrials.filter((t) => t.state !== "Complete") - } - if (filterPrunedTrial) { - filteredTrials = filteredTrials.filter((t) => t.state !== "Pruned") - } - if (filteredTrials.length === 0) { + if (trials.length === 0) { plotly.react(plotDomId, [], layout) return } @@ -320,10 +226,8 @@ const plotHistory = ( const plotData: Partial[] = [ { - x: filteredTrials.map(getAxisX), - y: filteredTrials.map( - (t: Trial): number => target.getTargetValue(t) as number - ), + x: trials.map(getAxisX), + y: trials.map((t: Trial): number => target.getTargetValue(t) as number), name: "Objective Value", mode: "markers", type: "scatter", @@ -335,17 +239,17 @@ const plotHistory = ( const xForLinePlot: (number | Date)[] = [] const yForLinePlot: number[] = [] let currentBest: number | null = null - for (let i = 0; i < filteredTrials.length; i++) { - const t = filteredTrials[i] + for (let i = 0; i < trials.length; i++) { + const t = trials[i] if (currentBest === null) { currentBest = t.values![objectiveId] as number xForLinePlot.push(getAxisX(t)) yForLinePlot.push(t.values![objectiveId] as number) } else if ( - study.directions[objectiveId] === "maximize" && + directions[objectiveId] === "maximize" && t.values![objectiveId] > currentBest ) { - const p = filteredTrials[i - 1] + const p = trials[i - 1] if (!xForLinePlot.includes(getAxisX(p))) { xForLinePlot.push(getAxisX(p)) yForLinePlot.push(currentBest) @@ -354,10 +258,10 @@ const plotHistory = ( xForLinePlot.push(getAxisX(t)) yForLinePlot.push(t.values![objectiveId] as number) } else if ( - study.directions[objectiveId] === "minimize" && + directions[objectiveId] === "minimize" && t.values![objectiveId] < currentBest ) { - const p = filteredTrials[i - 1] + const p = trials[i - 1] if (!xForLinePlot.includes(getAxisX(p))) { xForLinePlot.push(getAxisX(p)) yForLinePlot.push(currentBest) @@ -367,7 +271,7 @@ const plotHistory = ( yForLinePlot.push(t.values![objectiveId] as number) } } - xForLinePlot.push(getAxisX(filteredTrials[filteredTrials.length - 1])) + xForLinePlot.push(getAxisX(trials[trials.length - 1])) yForLinePlot.push(yForLinePlot[yForLinePlot.length - 1]) plotData.push({ x: xForLinePlot, diff --git a/optuna_dashboard/ts/trialFilter.ts b/optuna_dashboard/ts/trialFilter.ts new file mode 100644 index 00000000..9952411c --- /dev/null +++ b/optuna_dashboard/ts/trialFilter.ts @@ -0,0 +1,112 @@ +import { useMemo } from "react" + +export class Target { + kind: "objective" | "user_attr" + key: number | string + + constructor(kind: "objective" | "user_attr", key: number | string) { + this.kind = kind + this.key = key + } + + validate(): boolean { + if (this.kind === "objective") { + if (typeof this.key !== "number") { + return false + } + } else if (this.kind === "user_attr") { + if (typeof this.key !== "string") { + return false + } + } else { + return false + } + return true + } + + toLabel(objectiveNames: string[]): string { + if (this.kind === "objective") { + const objectiveId: number = this.key as number + if (objectiveNames.length > objectiveId) { + return objectiveNames[objectiveId] + } + return `Objective ${objectiveId}` + } else { + return `User Attribute ${this.key}` + } + } + + getObjectiveId(): number | null { + return this.key as number + } + + getTargetValue(trial: Trial): number | null { + if (!this.validate()) { + return null + } + if (this.kind === "objective") { + const objectiveId = this.getObjectiveId() + if ( + objectiveId === null || + trial.values === undefined || + trial.values.length <= objectiveId + ) { + return null + } + const value = trial.values[objectiveId] + if (value === "inf" || value === "-inf") { + return null + } + return value + } else if (this.kind === "user_attr") { + const attr = trial.user_attrs.find((attr) => attr.key === this.key) + if (attr === undefined) { + return null + } + const value = Number(attr.value) + if (value === undefined) { + return null + } + return value + } + return null + } +} + +export const useFilteredTrials = ( + study: StudyDetail | null, + target: Target, + filterComplete: boolean, + filterPruned: boolean +): Trial[] => + useMemo(() => { + if (study === null) { + return [] + } + return study.trials.filter((t) => { + if (t.state !== "Complete" && t.state !== "Pruned") { + return false + } + if (t.state === "Complete" && filterComplete) { + return false + } + if (t.state === "Pruned" && filterPruned) { + return false + } + return target.getTargetValue(t) !== null + }) + }, [study?.trials, target, filterComplete, filterPruned]) + +export const useTargetList = (study: StudyDetail | null): Target[] => + useMemo(() => { + if (study !== null) { + return [ + ...study.directions.map((v, i) => new Target("objective", i)), + ...study.union_user_attrs + .filter((attr) => attr.sortable) + .map((attr) => new Target("user_attr", attr.key)), + ] + } else { + return [new Target("objective", 0)] + } + }, [study?.directions, study?.union_user_attrs]) From 39f8637abc2929d9c0a3facc3f31bc12153917d5 Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 6 Jan 2023 20:23:32 +0900 Subject: [PATCH 02/14] Normalize distributions --- optuna_dashboard/_serializer.py | 38 ++++++++++++++++++++++++++++ optuna_dashboard/ts/types/index.d.ts | 5 ---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 6442fa91..f3e405d4 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -7,6 +7,8 @@ from typing import Union import numpy as np from optuna.distributions import BaseDistribution +from optuna.distributions import FloatDistribution +from optuna.distributions import IntDistribution from optuna.study import StudySummary from optuna.trial import FrozenTrial @@ -158,11 +160,47 @@ def serialize_frozen_trial( return serialized +def normalize_distribution(distribution: BaseDistribution) -> BaseDistribution: + if distribution.__class__.__name__ == "UniformDistribution": + return FloatDistribution( + low=getattr(distribution, "low"), + high=getattr(distribution, "high"), + ) + elif distribution.__class__.__name__ == "LogUniformDistribution": + return FloatDistribution( + low=getattr(distribution, "low"), + high=getattr(distribution, "high"), + log=True, + ) + elif distribution.__class__.__name__ == "DiscreteUniformDistribution": + return FloatDistribution( + low=getattr(distribution, "low"), + high=getattr(distribution, "high"), + step=getattr(distribution, "q") + ) + elif distribution.__class__.__name__ == "IntUniformDistribution": + return IntDistribution( + low=getattr(distribution, "low"), + high=getattr(distribution, "high"), + step=getattr(distribution, "step") + ) + elif distribution.__class__.__name__ == "IntLogUniformDistribution": + return IntDistribution( + low=getattr(distribution, "low"), + high=getattr(distribution, "high"), + step=getattr(distribution, "step"), + log=True + ) + else: + return distribution + + def serialize_search_space( search_space: list[tuple[str, BaseDistribution]] ) -> list[dict[str, Any]]: serialized = [] for param_name, distribution in search_space: + distribution = normalize_distribution(distribution) serialized.append( { "name": param_name, diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 98c42ed7..2c71848a 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -14,11 +14,6 @@ type StudyDirection = "maximize" | "minimize" | "not_set" type Distribution = | "FloatDistribution" | "IntDistribution" - | "UniformDistribution" - | "LogUniformDistribution" - | "DiscreteUniformDistribution" - | "IntUniformDistribution" - | "IntLogUniformDistribution" | "CategoricalDistribution" type PageId = "history" | "analytics" | "trialTable" | "trialList" | "note" From 744444d6990696196982c6798080ed8fe81189dd Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 6 Jan 2023 20:27:57 +0900 Subject: [PATCH 03/14] Move PageId definition to AppDrawer --- optuna_dashboard/ts/components/AppDrawer.tsx | 7 +++++++ optuna_dashboard/ts/components/StudyDetailBeta.tsx | 2 +- optuna_dashboard/ts/types/index.d.ts | 2 -- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index bc889c7e..99306435 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -33,6 +33,13 @@ import { Switch } from "@mui/material" const drawerWidth = 240 +export type PageId = + | "history" + | "analytics" + | "trialTable" + | "trialList" + | "note" + const openedMixin = (theme: Theme): CSSObject => ({ width: drawerWidth, transition: theme.transitions.create("width", { diff --git a/optuna_dashboard/ts/components/StudyDetailBeta.tsx b/optuna_dashboard/ts/components/StudyDetailBeta.tsx index 2152cf90..a98c3d04 100644 --- a/optuna_dashboard/ts/components/StudyDetailBeta.tsx +++ b/optuna_dashboard/ts/components/StudyDetailBeta.tsx @@ -24,7 +24,7 @@ import { useStudySummaryValue, } from "../state" import { TrialTable } from "./TrialTable" -import { AppDrawer } from "./AppDrawer" +import { AppDrawer, PageId } from "./AppDrawer" import { GraphParallelCoordinate } from "./GraphParallelCoordinate" import { Contour } from "./GraphContour" import { GraphHyperparameterImportanceBeta } from "./GraphHyperparameterImportances" diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 2c71848a..425045a2 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -16,8 +16,6 @@ type Distribution = | "IntDistribution" | "CategoricalDistribution" -type PageId = "history" | "analytics" | "trialTable" | "trialList" | "note" - type GraphVisibility = { history: boolean paretoFront: boolean From e77c8daf06e8f7a84bc58af70917ed5216f71d6c Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 6 Jan 2023 23:56:07 +0900 Subject: [PATCH 04/14] Use distributions for plots --- optuna_dashboard/_serializer.py | 85 +++++++- optuna_dashboard/ts/apiClient.ts | 4 +- .../ts/components/BestTrialsCard.tsx | 7 +- .../ts/components/GraphContour.tsx | 205 ++++++++++-------- .../ts/components/GraphHistory.tsx | 10 +- .../ts/components/GraphParallelCoordinate.tsx | 19 +- .../ts/components/GraphParetoFront.tsx | 2 +- optuna_dashboard/ts/components/GraphSlice.tsx | 149 ++++++------- optuna_dashboard/ts/components/TrialList.tsx | 5 +- optuna_dashboard/ts/components/TrialTable.tsx | 17 +- optuna_dashboard/ts/trialFilter.ts | 44 +++- optuna_dashboard/ts/types/index.d.ts | 39 +++- typescript_tests/TrialTable.test.tsx | 8 +- 13 files changed, 373 insertions(+), 221 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index f3e405d4..9503a75a 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING from typing import Union import numpy as np -from optuna.distributions import BaseDistribution +from optuna.distributions import BaseDistribution, CategoricalDistribution from optuna.distributions import FloatDistribution from optuna.distributions import IntDistribution from optuna.study import StudySummary @@ -42,6 +42,41 @@ if TYPE_CHECKING: }, ) + FloatDistributionJSON = TypedDict( + "FloatDistributionJSON", + { + "type": Literal["FloatDistribution"], + "low": float, + "high": float, + "step": float, + "log": bool, + }, + ) + IntDistributionJSON = TypedDict( + "IntDistributionJSON", + { + "type": Literal["IntDistribution"], + "low": int, + "high": int, + "step": int, + "log": bool, + }, + ) + CategoricalDistributionChoiceJSON = TypedDict( + "CategoricalDistributionChoiceJSON", + { + "pytype": str, + "value": str, + } + ) + CategoricalDistributionJSON = TypedDict( + "CategoricalDistributionJSON", + { + "choices": list[CategoricalDistributionChoiceJSON] + }, + ) + DistributionJSON = Union[FloatDistributionJSON, IntDistributionJSON, CategoricalDistributionJSON] + MAX_ATTR_LENGTH = 1024 @@ -111,12 +146,22 @@ def serialize_study_detail( def serialize_frozen_trial( study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any] ) -> dict[str, Any]: + params = [] + for param_name, param_external_value in trial.params.items(): + distribution = trial.distributions[param_name] + params.append({ + "name": param_name, + "param_internal_value": distribution.to_internal_repr(param_external_value), + "param_external_value": str(param_external_value), + "param_external_pytyp": str(type(param_external_value)), + "distribution": serialize_distribution(distribution) + }) serialized = { "trial_id": trial._trial_id, "study_id": study_id, "number": trial.number, "state": trial.state.name.capitalize(), - "params": [{"name": name, "value": str(value)} for name, value in trial.params.items()], + "params": params, "user_attrs": serialize_attrs(trial.user_attrs), "system_attrs": serialize_attrs(getattr(trial, "_system_attrs", {})), "note": note.get_note_from_system_attrs(study_system_attrs, trial._trial_id), @@ -160,6 +205,38 @@ def serialize_frozen_trial( return serialized +def serialize_distribution(distribution: BaseDistribution) -> DistributionJSON: + distribution = normalize_distribution(distribution) + if isinstance(distribution, FloatDistribution): + return { + "type": "FloatDistribution", + "low": distribution.low, + "high": distribution.high, + "step": distribution.step, + "log": distribution.log, + } + if isinstance(distribution, IntDistribution): + return { + "type": "IntDistribution", + "low": distribution.low, + "high": distribution.high, + "step": distribution.step, + "log": distribution.log, + } + if isinstance(distribution, CategoricalDistribution): + return { + "type": "CategoricalDistribution", + "choices": [ + { + "pytype": str(type(choice)), + "value": str(choice) + } + for choice in distribution.choices + ], + } + raise ValueError(f"Unexpected distribution {str(distribution)}") + + def normalize_distribution(distribution: BaseDistribution) -> BaseDistribution: if distribution.__class__.__name__ == "UniformDistribution": return FloatDistribution( @@ -200,12 +277,10 @@ def serialize_search_space( ) -> list[dict[str, Any]]: serialized = [] for param_name, distribution in search_space: - distribution = normalize_distribution(distribution) serialized.append( { "name": param_name, - "distribution": distribution.__class__.__name__, - "attributes": distribution._asdict(), + "distribution": serialize_distribution(distribution), } ) return serialized diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index f98cbd18..5b3fc2fe 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -44,8 +44,8 @@ interface StudyDetailResponse { directions: StudyDirection[] trials: TrialResponse[] best_trials: TrialResponse[] - intersection_search_space: SearchSpace[] - union_search_space: SearchSpace[] + intersection_search_space: SearchSpaceItem[] + union_search_space: SearchSpaceItem[] union_user_attrs: AttributeSpec[] has_intermediate_values: boolean note: Note diff --git a/optuna_dashboard/ts/components/BestTrialsCard.tsx b/optuna_dashboard/ts/components/BestTrialsCard.tsx index c1a5d7b4..a76802e8 100644 --- a/optuna_dashboard/ts/components/BestTrialsCard.tsx +++ b/optuna_dashboard/ts/components/BestTrialsCard.tsx @@ -41,7 +41,10 @@ export const BestTrialsCard: FC<{ Params = [ - {bestTrial.params.map((p) => `${p.name}: ${p.value}`).join(", ")}] + {bestTrial.params + .map((p) => `${p.name}: ${p.param_external_value}`) + .join(", ")} + ] Intermediate Values = [ @@ -101,7 +104,7 @@ export const BestTrialsCard: FC<{ Params = [ {trial.params - .map((p) => `${p.name}: ${p.value}`) + .map((p) => `${p.name}: ${p.param_external_value}`) .join(", ")} ] diff --git a/optuna_dashboard/ts/components/GraphContour.tsx b/optuna_dashboard/ts/components/GraphContour.tsx index b8301e20..06cb233b 100644 --- a/optuna_dashboard/ts/components/GraphContour.tsx +++ b/optuna_dashboard/ts/components/GraphContour.tsx @@ -1,5 +1,5 @@ import * as plotly from "plotly.js-dist-min" -import React, { FC, useEffect, useState } from "react" +import React, { FC, useEffect, useMemo, useState } from "react" import { Grid, FormControl, @@ -33,31 +33,44 @@ type AxisInfo = { const PADDING_RATIO = 0.05 const plotDomId = "graph-contour" +const useSearchSpace = ( + unionSearchSpaces?: SearchSpaceItem[] +): SearchSpaceItem[] => + useMemo( + () => + Array.from(unionSearchSpaces || []).sort((a, b) => + a.name > b.name ? 1 : a.name < b.name ? -1 : 0 + ), + [unionSearchSpaces] + ) + export const Contour: FC<{ study: StudyDetail | null }> = ({ study = null }) => { const theme = useTheme() const [objectiveId, setObjectiveId] = useState(0) - const [xParam, setXParam] = useState("") - const [yParam, setYParam] = useState("") - const paramNames = study?.union_search_space.map((s) => s.name) + const searchSpaces = useSearchSpace(study?.union_search_space) + const [xParam, setXParam] = useState(null) + const [yParam, setYParam] = useState(null) const objectiveNames: string[] = study?.objective_names || [] - if (!xParam && paramNames && paramNames.length > 0) { - setXParam(paramNames[0]) + if (xParam === null && searchSpaces.length > 0) { + setXParam(searchSpaces[0]) } - if (!yParam && paramNames && paramNames.length > 1) { - setYParam(paramNames[1]) + if (yParam === null && searchSpaces.length > 1) { + setYParam(searchSpaces[1]) } const handleObjectiveChange = (event: SelectChangeEvent) => { setObjectiveId(event.target.value as number) } const handleXParamChange = (event: SelectChangeEvent) => { - setXParam(event.target.value as string) + const param = searchSpaces.find((s) => s.name === event.target.value) + setXParam(param || null) } const handleYParamChange = (event: SelectChangeEvent) => { - setYParam(event.target.value as string) + const param = searchSpaces.find((s) => s.name === event.target.value) + setYParam(param || null) } useEffect(() => { @@ -66,7 +79,7 @@ export const Contour: FC<{ } }, [study, objectiveId, xParam, yParam, theme.palette.mode]) - const space: SearchSpace[] = study ? study.union_search_space : [] + const space: SearchSpaceItem[] = study ? study.union_search_space : [] return ( @@ -98,7 +111,7 @@ export const Contour: FC<{ x: - {space.map((d, i) => ( {d.name} @@ -108,7 +121,7 @@ export const Contour: FC<{ y: - {space.map((d, i) => ( {d.name} @@ -126,73 +139,6 @@ export const Contour: FC<{ ) } -const isNumerical = (trials: Trial[], paramName: string): boolean => { - return trials.every((t) => { - const param = t.params.find((param) => param.name === paramName) - if (!param) return true - const val = param.value - return typeof (Number(val) || val) === "number" - }) -} - -const getAxisInfo = (trials: Trial[], paramName: string): AxisInfo => { - const values = trials.map((trial) => { - const param = trial.params.find((p) => p.name === paramName) - return param ? Number(param.value) || param.value : null - }) - - let min: number - let max: number - let isLog: boolean - let isCat: boolean - - if (isNumerical(trials, paramName)) { - const minValue = Math.min(...(values as number[])) - const maxValue = Math.max(...(values as number[])) - const padding = (maxValue - minValue) * PADDING_RATIO - min = minValue - padding - max = maxValue + padding - isLog = false - isCat = false - } else { - const uniqueValues = unique(values) - const span = uniqueValues.length - (uniqueValues.includes(null) ? 2 : 1) - const padding = span * PADDING_RATIO - min = -padding - max = span + padding - isLog = false - isCat = true - } - - const indices = isNumerical(trials, paramName) - ? unique((values as (number | null)[]).filter((v) => v !== null)).sort( - (a, b) => a - b - ) - : unique((values as (string | null)[]).filter((v) => v !== null)).sort( - (a, b) => - a.toString().toLowerCase() < b.toString().toLowerCase() - ? -1 - : a.toString().toLowerCase() > b.toString().toLowerCase() - ? 1 - : 0 - ) - - if (indices.length >= 2 && isNumerical(trials, paramName)) { - indices.unshift(min) - indices.push(max) - } - - return { - name: paramName, - min, - max, - isLog, - isCat, - indices, - values, - } -} - const filterFunc = (trial: Trial, objectiveId: number): boolean => { return ( trial.state === "Complete" && @@ -205,8 +151,8 @@ const filterFunc = (trial: Trial, objectiveId: number): boolean => { const plotContour = ( study: StudyDetail, objectiveId: number, - xParam: string, - yParam: string, + xParam: SearchSpaceItem | null, + yParam: SearchSpaceItem | null, mode: string ) => { if (document.getElementById(plotDomId) === null) { @@ -215,16 +161,15 @@ const plotContour = ( const trials: Trial[] = study ? study.trials : [] const filteredTrials = trials.filter((t) => filterFunc(t, objectiveId)) - - if (filteredTrials.length === 0) { + if (filteredTrials.length === 0 || xParam === null || yParam === null) { plotly.react(plotDomId, [], { template: mode === "dark" ? plotlyDarkTemplate : {}, }) return } - const xAxis = getAxisInfo(trials, xParam) - const yAxis = getAxisInfo(trials, yParam) + const xAxis = getAxisInfo(study, trials, xParam) + const yAxis = getAxisInfo(study, trials, yParam) const xIndices = xAxis.indices const yIndices = yAxis.indices @@ -279,11 +224,11 @@ const plotContour = ( const layout: Partial = { xaxis: { - title: xParam, + title: xParam.name, type: xAxis.isCat ? "category" : undefined, }, yaxis: { - title: yParam, + title: yParam.name, type: yAxis.isCat ? "category" : undefined, }, margin: { @@ -296,3 +241,87 @@ const plotContour = ( } plotly.react(plotDomId, plotData, layout) } + +const getAxisInfoForNumericalParams = ( + trials: Trial[], + paramName: string, + distribution: FloatDistribution | IntDistribution +): AxisInfo => { + const padding = (distribution.high - distribution.low) * PADDING_RATIO + const min = distribution.low - padding + const max = distribution.high + padding + + const values = trials.map( + (trial) => + trial.params.find((p) => p.name === paramName)?.param_internal_value || + null + ) + const indices = unique(values) + .filter((v) => v !== null) + .sort((a, b) => a - b) + if (indices.length >= 2) { + indices.unshift(min) + indices.push(max) + } + return { + name: paramName, + min, + max, + isLog: distribution.log, + isCat: false, + indices, + values, + } +} + +const getAxisInfoForCategoricalParams = ( + trials: Trial[], + paramName: string, + distribution: CategoricalDistribution +): AxisInfo => { + const values = trials.map( + (trial) => + trial.params.find((p) => p.name === paramName)?.param_external_value || + null + ) + const isDynamic = values.some((v) => v === null) + const span = distribution.choices.length - (isDynamic ? 2 : 1) + const padding = span * PADDING_RATIO + const min = -padding + const max = span + padding + + const indices = distribution.choices + .map((c) => c.value) + .sort((a, b) => + a.toLowerCase() < b.toLowerCase() + ? -1 + : a.toLowerCase() > b.toLowerCase() + ? 1 + : 0 + ) + return { + name: paramName, + min, + max, + isLog: false, + isCat: true, + indices, + values, + } +} + +const getAxisInfo = ( + study: StudyDetail, + trials: Trial[], + param: SearchSpaceItem +): AxisInfo => { + if (param.distribution.type === "CategoricalDistribution") { + return getAxisInfoForCategoricalParams( + trials, + param.name, + param.distribution + ) + } else { + return getAxisInfoForNumericalParams(trials, param.name, param.distribution) + } +} diff --git a/optuna_dashboard/ts/components/GraphHistory.tsx b/optuna_dashboard/ts/components/GraphHistory.tsx index 4f55d735..8c38fc48 100644 --- a/optuna_dashboard/ts/components/GraphHistory.tsx +++ b/optuna_dashboard/ts/components/GraphHistory.tsx @@ -16,7 +16,11 @@ import { useTheme, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" -import { useFilteredTrials, Target, useTargetList } from "../trialFilter" +import { + useFilteredTrials, + Target, + useObjectiveAndSystemAttrTargets, +} from "../trialFilter" const plotDomId = "graph-history" @@ -30,11 +34,11 @@ export const GraphHistory: FC<{ const [filterPrunedTrial, setFilterPrunedTrial] = useState(false) const objectiveNames: string[] = study?.objective_names || [] - const targetList = useTargetList(study) + const targetList = useObjectiveAndSystemAttrTargets(study) const [targetIndex, setTargetIndex] = useState(0) const trials = useFilteredTrials( study, - targetList[targetIndex], + [targetList[targetIndex]], filterCompleteTrial, filterPrunedTrial ) diff --git a/optuna_dashboard/ts/components/GraphParallelCoordinate.tsx b/optuna_dashboard/ts/components/GraphParallelCoordinate.tsx index ca394e62..7347f42e 100644 --- a/optuna_dashboard/ts/components/GraphParallelCoordinate.tsx +++ b/optuna_dashboard/ts/components/GraphParallelCoordinate.tsx @@ -136,15 +136,10 @@ const plotCoordinate = ( }, ] study.intersection_search_space.forEach((s) => { - const valueStrings = filteredTrials.map((t) => { - const param = t.params.find((p) => p.name === s.name) - return param!.value - }) - const isnum = valueStrings.every((v) => { - return !isNaN(Number(v)) - }) - if (isnum) { - const values: number[] = valueStrings.map((v) => parseFloat(v)) + const values: number[] = filteredTrials.map( + (t) => t.params.find((p) => p.name === s.name)!.param_internal_value + ) + if (s.distribution.type !== "CategoricalDistribution") { dimensions.push({ label: breakLabelIfTooLong(s.name), values: values, @@ -152,11 +147,7 @@ const plotCoordinate = ( }) } else { // categorical - const vocabSet = new Set(valueStrings) - const vocabArr = Array.from(vocabSet) - const values: number[] = valueStrings.map((v) => - vocabArr.findIndex((vocab) => v === vocab) - ) + const vocabArr: string[] = s.distribution.choices.map((c) => c.value) const tickvals: number[] = vocabArr.map((v, i) => i) dimensions.push({ label: breakLabelIfTooLong(s.name), diff --git a/optuna_dashboard/ts/components/GraphParetoFront.tsx b/optuna_dashboard/ts/components/GraphParetoFront.tsx index ef3dd646..22c4007c 100644 --- a/optuna_dashboard/ts/components/GraphParetoFront.tsx +++ b/optuna_dashboard/ts/components/GraphParetoFront.tsx @@ -105,7 +105,7 @@ const makeHovertext = (trial: Trial): string => { number: trial.number, values: trial.values, params: trial.params - .map((p) => [p.name, p.value]) + .map((p) => [p.name, p.param_external_value]) .reduce((obj, [key, value]) => ({ ...obj, [key]: value }), {}), }, undefined, diff --git a/optuna_dashboard/ts/components/GraphSlice.tsx b/optuna_dashboard/ts/components/GraphSlice.tsx index 31231881..d1e201bf 100644 --- a/optuna_dashboard/ts/components/GraphSlice.tsx +++ b/optuna_dashboard/ts/components/GraphSlice.tsx @@ -1,5 +1,5 @@ import * as plotly from "plotly.js-dist-min" -import React, { ChangeEvent, FC, useEffect, useState } from "react" +import React, { ChangeEvent, FC, useEffect, useMemo, useState } from "react" import { Grid, FormControl, @@ -13,52 +13,76 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" +import { Target, useFilteredTrials, useObjectiveTargets } from "../trialFilter" +import { useSnackbar } from "notistack" const plotDomId = "graph-slice" -// TODO(c-bata): Check `log` field of IntDistribution and FloatDistribution. -const logDistributions = ["LogUniformDistribution", "IntLogUniformDistribution"] +const useSearchSpace = ( + unionSearchSpaces?: SearchSpaceItem[] +): SearchSpaceItem[] => + useMemo( + () => + Array.from(unionSearchSpaces || []).sort((a, b) => + a.name > b.name ? 1 : a.name < b.name ? -1 : 0 + ), + [unionSearchSpaces] + ) + +const isLogScale = (s: SearchSpaceItem): boolean => { + if (s.distribution.type === "CategoricalDistribution") { + return false + } + return s.distribution.log +} export const GraphSlice: FC<{ study: StudyDetail | null }> = ({ study = null }) => { const theme = useTheme() - const trials: Trial[] = study !== null ? study.trials : [] + const { enqueueSnackbar } = useSnackbar() + const [objectiveId, setObjectiveId] = useState(0) - const [selected, setSelected] = useState(null) - const [logXScale, setLogXScale] = useState(false) + const [selected, setSelected] = useState(null) const [logYScale, setLogYScale] = useState(false) - const paramNames = study?.union_search_space.map((s) => s.name) - const distributions = new Map( - study?.union_search_space.map((s) => [s.name, s.distribution]) - ) + const searchSpaces = useSearchSpace(study?.union_search_space) + + const targets = useObjectiveTargets(study) + const filterTargets: Target[] = [targets[objectiveId]] + if (selected !== null) filterTargets.push(new Target("params", selected.name)) + const trials = useFilteredTrials(study, filterTargets, false, false) + const objectiveNames: string[] = study?.objective_names || [] - if (selected === null && paramNames && paramNames.length > 0) { - const distribution = distributions.get(paramNames[0]) || "" - setSelected(paramNames[0]) - setLogXScale(logDistributions.includes(distribution)) + if (selected === null && searchSpaces.length > 0) { + setSelected(searchSpaces[0]) } useEffect(() => { plotSlice( trials, - objectiveId, + targets[objectiveId], selected, - logXScale, logYScale, theme.palette.mode ) - }, [trials, objectiveId, selected, logXScale, logYScale, theme.palette.mode]) + }, [trials, targets[objectiveId], selected, logYScale, theme.palette.mode]) const handleObjectiveChange = (event: SelectChangeEvent) => { setObjectiveId(event.target.value as number) } const handleSelectedParam = (e: SelectChangeEvent) => { - const paramName = e.target.value - const distribution = distributions.get(paramName) || "" - setSelected(paramName) - setLogXScale(logDistributions.includes(distribution)) + const s = searchSpaces.find((s) => s.name === e.target.value) + if (s === undefined) { + enqueueSnackbar( + `Cannot find ${e.target.value} param in the search space.`, + { + variant: "error", + } + ) + return + } + setSelected(s) } const handleLogYScaleChange = (e: ChangeEvent) => { @@ -93,9 +117,9 @@ export const GraphSlice: FC<{ )} Parameter: - + {searchSpaces?.map((s, i) => ( + {objectiveNames.length === study?.directions.length ? objectiveNames[i] : `${i}`} @@ -119,32 +143,10 @@ export const GraphSlice: FC<{ ) } -const filterFunc = ( - trial: Trial, - objectiveId: number, - selected: string | null -): boolean => { - if (trial.state !== "Complete" && trial.state !== "Pruned") { - return false - } - if (trial.params.find((p) => p.name == selected) === undefined) { - return false - } - if (trial.values === undefined) { - return false - } - return ( - trial.values.length > objectiveId && - trial.values[objectiveId] !== "inf" && - trial.values[objectiveId] !== "-inf" - ) -} - const plotSlice = ( trials: Trial[], - objectiveId: number, - selected: string | null, - logXScale: boolean, + target: Target, + selected: SearchSpaceItem | null, logYScale: boolean, mode: string ) => { @@ -160,8 +162,8 @@ const plotSlice = ( b: 0, }, xaxis: { - title: selected || "", - type: logXScale ? "log" : "linear", + title: selected?.name || "", + type: selected !== null && isLogScale(selected) ? "log" : "linear", gridwidth: 1, automargin: true, }, @@ -174,34 +176,27 @@ const plotSlice = ( showlegend: false, template: mode === "dark" ? plotlyDarkTemplate : {}, } - - const filteredTrials = trials.filter((t) => - filterFunc(t, objectiveId, selected) - ) - - if (filteredTrials.length === 0 || selected === null) { + if (selected === null) { + plotly.react(plotDomId, [], layout) + return + } + if (trials.length === 0) { plotly.react(plotDomId, [], layout) return } - const objectiveValues: number[] = filteredTrials.map( - (t) => t.values![objectiveId] as number + const objectiveValues: number[] = trials.map( + (t) => target.getTargetValue(t) as number ) - const valueStrings = filteredTrials.map((t) => { - return t.params.find((p) => p.name == selected)!.value - }) + const paramTarget = new Target("params", selected.name) + const values = trials.map((t) => paramTarget.getTargetValue(t) as number) - const trialNumbers: number[] = filteredTrials.map((t) => t.number) - - const isnum = valueStrings.every((v) => { - return !isNaN(Number(v)) - }) - if (isnum) { - const valuesNum: number[] = valueStrings.map((v) => parseFloat(v)) + const trialNumbers: number[] = trials.map((t) => t.number) + if (selected.distribution.type !== "CategoricalDistribution") { const trace: plotly.Data[] = [ { type: "scatter", - x: valuesNum, + x: values, y: objectiveValues, mode: "markers", marker: { @@ -219,23 +214,19 @@ const plotSlice = ( }, ] layout["xaxis"] = { - title: selected, - type: logXScale ? "log" : "linear", + title: selected.name, + type: selected.distribution.log ? "log" : "linear", gridwidth: 1, automargin: true, // Otherwise the label is outside of the plot } plotly.react(plotDomId, trace, layout) } else { - const vocabSet = new Set(valueStrings) - const vocabArr = Array.from(vocabSet) - const valuesCategorical: number[] = valueStrings.map((v) => - vocabArr.findIndex((vocab) => v === vocab) - ) + const vocabArr = selected.distribution.choices.map((c) => c.value) const tickvals: number[] = vocabArr.map((v, i) => i) const trace: plotly.Data[] = [ { type: "scatter", - x: valuesCategorical, + x: values, y: objectiveValues, mode: "markers", marker: { @@ -253,8 +244,8 @@ const plotSlice = ( }, ] layout["xaxis"] = { - title: selected, - type: logXScale ? "log" : "linear", + title: selected.name, + type: "linear", gridwidth: 1, tickvals: tickvals, ticktext: vocabArr, diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 44b180c7..8e20df96 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -162,7 +162,10 @@ const TrialListDetail: FC<{ Params = [ - {trial.params.map((p) => `${p.name}: ${p.value}`).join(", ")}] + {trial.params + .map((p) => `${p.name}: ${p.param_external_value}`) + .join(", ")} + ] Started At ={" "} diff --git a/optuna_dashboard/ts/components/TrialTable.tsx b/optuna_dashboard/ts/components/TrialTable.tsx index ba271bf1..f3a55922 100644 --- a/optuna_dashboard/ts/components/TrialTable.tsx +++ b/optuna_dashboard/ts/components/TrialTable.tsx @@ -139,20 +139,23 @@ export const TrialTable: FC<{ studyDetail?.intersection_search_space.length ) { studyDetail?.intersection_search_space.forEach((s) => { - const sortable = s.distribution !== "CategoricalDistribution" - const filterable = s.distribution === "CategoricalDistribution" + const sortable = s.distribution.type !== "CategoricalDistribution" + const filterable = s.distribution.type === "CategoricalDistribution" columns.push({ field: "params", label: `Param ${s.name}`, toCellValue: (i) => - trials[i].params.find((p) => p.name === s.name)?.value || null, + trials[i].params.find((p) => p.name === s.name) + ?.param_external_value || null, sortable: sortable, filterable: filterable, less: (firstEl, secondEl): number => { - const firstVal = firstEl.params.find((p) => p.name === s.name)?.value + const firstVal = firstEl.params.find( + (p) => p.name === s.name + )?.param_internal_value const secondVal = secondEl.params.find( (p) => p.name === s.name - )?.value + )?.param_internal_value if (firstVal === secondVal) { return 0 @@ -171,7 +174,9 @@ export const TrialTable: FC<{ field: "params", label: "Params", toCellValue: (i) => - trials[i].params.map((p) => p.name + ": " + p.value).join(", "), + trials[i].params + .map((p) => p.name + ": " + p.param_external_value) + .join(", "), }) } diff --git a/optuna_dashboard/ts/trialFilter.ts b/optuna_dashboard/ts/trialFilter.ts index 9952411c..1e249761 100644 --- a/optuna_dashboard/ts/trialFilter.ts +++ b/optuna_dashboard/ts/trialFilter.ts @@ -1,10 +1,12 @@ import { useMemo } from "react" +type TargetKind = "objective" | "user_attr" | "params" + export class Target { - kind: "objective" | "user_attr" + kind: TargetKind key: number | string - constructor(kind: "objective" | "user_attr", key: number | string) { + constructor(kind: TargetKind, key: number | string) { this.kind = kind this.key = key } @@ -18,8 +20,10 @@ export class Target { if (typeof this.key !== "string") { return false } - } else { - return false + } else if (this.kind === "params") { + if (typeof this.key !== "string") { + return false + } } return true } @@ -31,12 +35,17 @@ export class Target { return objectiveNames[objectiveId] } return `Objective ${objectiveId}` - } else { + } else if (this.kind === "user_attr") { return `User Attribute ${this.key}` + } else { + return `Param ${this.key}` } } getObjectiveId(): number | null { + if (this.kind !== "objective") { + return null + } return this.key as number } @@ -68,6 +77,12 @@ export class Target { return null } return value + } else if (this.kind === "params") { + const param = trial.params.find((p) => p.name === this.key) + if (param === undefined) { + return null + } + return param.param_internal_value } return null } @@ -75,7 +90,7 @@ export class Target { export const useFilteredTrials = ( study: StudyDetail | null, - target: Target, + targets: Target[], filterComplete: boolean, filterPruned: boolean ): Trial[] => @@ -93,11 +108,22 @@ export const useFilteredTrials = ( if (t.state === "Pruned" && filterPruned) { return false } - return target.getTargetValue(t) !== null + return targets.every((target) => target.getTargetValue(t) !== null) }) - }, [study?.trials, target, filterComplete, filterPruned]) + }, [study?.trials, targets, filterComplete, filterPruned]) -export const useTargetList = (study: StudyDetail | null): Target[] => +export const useObjectiveTargets = (study: StudyDetail | null): Target[] => + useMemo(() => { + if (study !== null) { + return study.directions.map((v, i) => new Target("objective", i)) + } else { + return [new Target("objective", 0)] + } + }, [study?.directions]) + +export const useObjectiveAndSystemAttrTargets = ( + study: StudyDetail | null +): Target[] => useMemo(() => { if (study !== null) { return [ diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 425045a2..440789ba 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -11,10 +11,32 @@ type TrialValueNumber = number | "inf" | "-inf" type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type StudyDirection = "maximize" | "minimize" | "not_set" + +type FloatDistribution = { + type: "FloatDistribution" + low: number + high: number + step: number + log: boolean +} + +type IntDistribution = { + type: "IntDistribution" + low: number + high: number + step: number + log: boolean +} + +type CategoricalDistribution = { + type: "CategoricalDistribution" + choices: { pytype: string; value: string }[] +} + type Distribution = - | "FloatDistribution" - | "IntDistribution" - | "CategoricalDistribution" + | FloatDistribution + | IntDistribution + | CategoricalDistribution type GraphVisibility = { history: boolean @@ -34,7 +56,10 @@ type TrialIntermediateValue = { type TrialParam = { name: string - value: string + param_internal_value: number + param_external_value: string + param_external_type: string + distribution: Distribution } type ParamImportance = { @@ -43,7 +68,7 @@ type ParamImportance = { distribution: Distribution } -type SearchSpace = { +type SearchSpaceItem = { name: string distribution: Distribution } @@ -94,8 +119,8 @@ type StudyDetail = { datetime_start: Date best_trials: Trial[] trials: Trial[] - intersection_search_space: SearchSpace[] - union_search_space: SearchSpace[] + intersection_search_space: SearchSpaceItem[] + union_search_space: SearchSpaceItem[] union_user_attrs: AttributeSpec[] has_intermediate_values: boolean note: Note diff --git a/typescript_tests/TrialTable.test.tsx b/typescript_tests/TrialTable.test.tsx index c443df13..df772179 100644 --- a/typescript_tests/TrialTable.test.tsx +++ b/typescript_tests/TrialTable.test.tsx @@ -61,21 +61,21 @@ const studyDetail: StudyDetail = { intersection_search_space: [ { name: "x", - distribution: "FloatDistribution" as Distribution, + distribution: "FloatDistribution" as DistributionName, }, { name: "y", - distribution: "FloatDistribution" as Distribution, + distribution: "FloatDistribution" as DistributionName, }, ], union_search_space: [ { name: "x", - distribution: "FloatDistribution" as Distribution, + distribution: "FloatDistribution" as DistributionName, }, { name: "y", - distribution: "FloatDistribution" as Distribution, + distribution: "FloatDistribution" as DistributionName, }, ], union_user_attrs: [ From 417b43e5c6989bba76b121211aa0f3b454f2d1d2 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 00:04:06 +0900 Subject: [PATCH 05/14] Fix DOM warnings --- .../ts/components/BestTrialsCard.tsx | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/optuna_dashboard/ts/components/BestTrialsCard.tsx b/optuna_dashboard/ts/components/BestTrialsCard.tsx index a76802e8..d34007e5 100644 --- a/optuna_dashboard/ts/components/BestTrialsCard.tsx +++ b/optuna_dashboard/ts/components/BestTrialsCard.tsx @@ -91,26 +91,23 @@ export const BestTrialsCard: FC<{ URL_PREFIX + `/studies/${trial.study_id}/trials?numbers=${trial.number}` } + sx={{ flexDirection: "column", alignItems: "flex-start" }} > Trial {trial.number} } - secondary={ - <> - - Objective Values = [{trial.values?.join(", ")}] - - - Params = [ - {trial.params - .map((p) => `${p.name}: ${p.param_external_value}`) - .join(", ")} - ] - - - } /> + + Objective Values = [{trial.values?.join(", ")}] + + + Params = [ + {trial.params + .map((p) => `${p.name}: ${p.param_external_value}`) + .join(", ")} + ] + ))} From 75e7fd018afccb5264cf56d8f1c9900a90d8b1b7 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 00:08:37 +0900 Subject: [PATCH 06/14] Fix broken tests --- typescript_tests/TrialTable.test.tsx | 49 +++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/typescript_tests/TrialTable.test.tsx b/typescript_tests/TrialTable.test.tsx index df772179..b9cd6035 100644 --- a/typescript_tests/TrialTable.test.tsx +++ b/typescript_tests/TrialTable.test.tsx @@ -6,7 +6,14 @@ import { TrialTable } from "../optuna_dashboard/ts/components/TrialTable" afterEach(cleanup) -const trials = [ +const dummyDistribution: FloatDistribution = { + type: "FloatDistribution", + low: 0, + high: 10, + step: 1, + log: false, +} +const trials: Trial[] = [ { trial_id: 1, study_id: 0, @@ -17,8 +24,20 @@ const trials = [ datetime_start: new Date("2021-06-15T00:00:00"), datetime_complete: new Date("2021-06-15T00:00:01"), params: [ - { name: "x", value: "1" }, - { name: "y", value: "2" }, + { + name: "x", + param_internal_value: 1, + param_external_value: "1", + param_external_type: "float", + distribution: dummyDistribution, + }, + { + name: "y", + param_internal_value: 2, + param_external_value: "2", + param_external_type: "float", + distribution: dummyDistribution, + }, ], user_attrs: [], system_attrs: [], @@ -37,8 +56,20 @@ const trials = [ datetime_start: new Date("2021-06-15T00:00:01"), datetime_complete: new Date("2021-06-15T00:00:03"), params: [ - { name: "x", value: "2" }, - { name: "y", value: "1" }, + { + name: "x", + param_internal_value: 1, + param_external_value: "1", + param_external_type: "float", + distribution: dummyDistribution, + }, + { + name: "y", + param_internal_value: 2, + param_external_value: "2", + param_external_type: "float", + distribution: dummyDistribution, + }, ], user_attrs: [], system_attrs: [], @@ -61,21 +92,21 @@ const studyDetail: StudyDetail = { intersection_search_space: [ { name: "x", - distribution: "FloatDistribution" as DistributionName, + distribution: dummyDistribution, }, { name: "y", - distribution: "FloatDistribution" as DistributionName, + distribution: dummyDistribution, }, ], union_search_space: [ { name: "x", - distribution: "FloatDistribution" as DistributionName, + distribution: dummyDistribution, }, { name: "y", - distribution: "FloatDistribution" as DistributionName, + distribution: dummyDistribution, }, ], union_user_attrs: [ From 8ce7cf9d958e4b354a343095b430e9cc0917f603 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 00:11:53 +0900 Subject: [PATCH 07/14] Fix python lint errors --- optuna_dashboard/_serializer.py | 40 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 9503a75a..0d8e6920 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -6,7 +6,8 @@ from typing import TYPE_CHECKING from typing import Union import numpy as np -from optuna.distributions import BaseDistribution, CategoricalDistribution +from optuna.distributions import BaseDistribution +from optuna.distributions import CategoricalDistribution from optuna.distributions import FloatDistribution from optuna.distributions import IntDistribution from optuna.study import StudySummary @@ -67,15 +68,15 @@ if TYPE_CHECKING: { "pytype": str, "value": str, - } + }, ) CategoricalDistributionJSON = TypedDict( "CategoricalDistributionJSON", - { - "choices": list[CategoricalDistributionChoiceJSON] - }, + {"choices": list[CategoricalDistributionChoiceJSON]}, ) - DistributionJSON = Union[FloatDistributionJSON, IntDistributionJSON, CategoricalDistributionJSON] + DistributionJSON = Union[ + FloatDistributionJSON, IntDistributionJSON, CategoricalDistributionJSON + ] MAX_ATTR_LENGTH = 1024 @@ -149,13 +150,15 @@ def serialize_frozen_trial( params = [] for param_name, param_external_value in trial.params.items(): distribution = trial.distributions[param_name] - params.append({ - "name": param_name, - "param_internal_value": distribution.to_internal_repr(param_external_value), - "param_external_value": str(param_external_value), - "param_external_pytyp": str(type(param_external_value)), - "distribution": serialize_distribution(distribution) - }) + params.append( + { + "name": param_name, + "param_internal_value": distribution.to_internal_repr(param_external_value), + "param_external_value": str(param_external_value), + "param_external_pytyp": str(type(param_external_value)), + "distribution": serialize_distribution(distribution), + } + ) serialized = { "trial_id": trial._trial_id, "study_id": study_id, @@ -227,10 +230,7 @@ def serialize_distribution(distribution: BaseDistribution) -> DistributionJSON: return { "type": "CategoricalDistribution", "choices": [ - { - "pytype": str(type(choice)), - "value": str(choice) - } + {"pytype": str(type(choice)), "value": str(choice)} for choice in distribution.choices ], } @@ -253,20 +253,20 @@ def normalize_distribution(distribution: BaseDistribution) -> BaseDistribution: return FloatDistribution( low=getattr(distribution, "low"), high=getattr(distribution, "high"), - step=getattr(distribution, "q") + step=getattr(distribution, "q"), ) elif distribution.__class__.__name__ == "IntUniformDistribution": return IntDistribution( low=getattr(distribution, "low"), high=getattr(distribution, "high"), - step=getattr(distribution, "step") + step=getattr(distribution, "step"), ) elif distribution.__class__.__name__ == "IntLogUniformDistribution": return IntDistribution( low=getattr(distribution, "low"), high=getattr(distribution, "high"), step=getattr(distribution, "step"), - log=True + log=True, ) else: return distribution From c007b89e30525029e6e7dedaf50d9bb75c5beb1b Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 00:25:40 +0900 Subject: [PATCH 08/14] Fix broken tests at Optuna 2.10 --- optuna_dashboard/_serializer.py | 107 ++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 0d8e6920..a20f6f01 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -8,8 +8,6 @@ from typing import Union import numpy as np from optuna.distributions import BaseDistribution from optuna.distributions import CategoricalDistribution -from optuna.distributions import FloatDistribution -from optuna.distributions import IntDistribution from optuna.study import StudySummary from optuna.trial import FrozenTrial @@ -209,23 +207,71 @@ def serialize_frozen_trial( def serialize_distribution(distribution: BaseDistribution) -> DistributionJSON: - distribution = normalize_distribution(distribution) - if isinstance(distribution, FloatDistribution): + if distribution.__class__.__name__ == "FloatDistribution": + # Added from Optuna v3.0 return { "type": "FloatDistribution", - "low": distribution.low, - "high": distribution.high, - "step": distribution.step, - "log": distribution.log, + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "step"), + "log": getattr(distribution, "log"), } - if isinstance(distribution, IntDistribution): + if distribution.__class__.__name__ == "UniformDistribution": + # Deprecated from Optuna v3.0 + return { + "type": "FloatDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": 0, + "log": False, + } + if distribution.__class__.__name__ == "LogUniformDistribution": + # Deprecated from Optuna v3.0 + return { + "type": "FloatDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": 0, + "log": True, + } + if distribution.__class__.__name__ == "DiscreteUniformDistribution": + # Deprecated from Optuna v3.0 + return { + "type": "FloatDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "q"), + "log": False, + } + + if distribution.__class__.__name__ == "IntDistribution": + # Added from Optuna v3.0 return { "type": "IntDistribution", - "low": distribution.low, - "high": distribution.high, - "step": distribution.step, - "log": distribution.log, + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "step"), + "log": getattr(distribution, "log"), } + if distribution.__class__.__name__ == "IntUniformDistribution": + # Deprecated from Optuna v3.0 + return { + "type": "IntDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "step"), + "log": False, + } + if distribution.__class__.__name__ == "IntLogUniformDistribution": + # Deprecated from Optuna v3.0 + return { + "type": "IntDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "step"), + "log": True, + } + if isinstance(distribution, CategoricalDistribution): return { "type": "CategoricalDistribution", @@ -237,41 +283,6 @@ def serialize_distribution(distribution: BaseDistribution) -> DistributionJSON: raise ValueError(f"Unexpected distribution {str(distribution)}") -def normalize_distribution(distribution: BaseDistribution) -> BaseDistribution: - if distribution.__class__.__name__ == "UniformDistribution": - return FloatDistribution( - low=getattr(distribution, "low"), - high=getattr(distribution, "high"), - ) - elif distribution.__class__.__name__ == "LogUniformDistribution": - return FloatDistribution( - low=getattr(distribution, "low"), - high=getattr(distribution, "high"), - log=True, - ) - elif distribution.__class__.__name__ == "DiscreteUniformDistribution": - return FloatDistribution( - low=getattr(distribution, "low"), - high=getattr(distribution, "high"), - step=getattr(distribution, "q"), - ) - elif distribution.__class__.__name__ == "IntUniformDistribution": - return IntDistribution( - low=getattr(distribution, "low"), - high=getattr(distribution, "high"), - step=getattr(distribution, "step"), - ) - elif distribution.__class__.__name__ == "IntLogUniformDistribution": - return IntDistribution( - low=getattr(distribution, "low"), - high=getattr(distribution, "high"), - step=getattr(distribution, "step"), - log=True, - ) - else: - return distribution - - def serialize_search_space( search_space: list[tuple[str, BaseDistribution]] ) -> list[dict[str, Any]]: From a25ff887ee2e78a681daa69098db98e2a091d22e Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 00:53:20 +0900 Subject: [PATCH 09/14] Fix mypy errors --- optuna_dashboard/_serializer.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index a20f6f01..1c26810c 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -70,7 +70,10 @@ if TYPE_CHECKING: ) CategoricalDistributionJSON = TypedDict( "CategoricalDistributionJSON", - {"choices": list[CategoricalDistributionChoiceJSON]}, + { + "type": Literal["CategoricalDistribution"], + "choices": list[CategoricalDistributionChoiceJSON], + }, ) DistributionJSON = Union[ FloatDistributionJSON, IntDistributionJSON, CategoricalDistributionJSON @@ -209,77 +212,83 @@ def serialize_frozen_trial( def serialize_distribution(distribution: BaseDistribution) -> DistributionJSON: if distribution.__class__.__name__ == "FloatDistribution": # Added from Optuna v3.0 - return { + float_distribution: FloatDistributionJSON = { "type": "FloatDistribution", "low": getattr(distribution, "low"), "high": getattr(distribution, "high"), "step": getattr(distribution, "step"), "log": getattr(distribution, "log"), } + return float_distribution if distribution.__class__.__name__ == "UniformDistribution": # Deprecated from Optuna v3.0 - return { + uniform: FloatDistributionJSON = { "type": "FloatDistribution", "low": getattr(distribution, "low"), "high": getattr(distribution, "high"), "step": 0, "log": False, } + return uniform if distribution.__class__.__name__ == "LogUniformDistribution": # Deprecated from Optuna v3.0 - return { + log_uniform: FloatDistributionJSON = { "type": "FloatDistribution", "low": getattr(distribution, "low"), "high": getattr(distribution, "high"), "step": 0, "log": True, } + return log_uniform if distribution.__class__.__name__ == "DiscreteUniformDistribution": # Deprecated from Optuna v3.0 - return { + discrete_uniform: FloatDistributionJSON = { "type": "FloatDistribution", "low": getattr(distribution, "low"), "high": getattr(distribution, "high"), "step": getattr(distribution, "q"), "log": False, } - + return discrete_uniform if distribution.__class__.__name__ == "IntDistribution": # Added from Optuna v3.0 - return { + int_distribution: IntDistributionJSON = { "type": "IntDistribution", "low": getattr(distribution, "low"), "high": getattr(distribution, "high"), "step": getattr(distribution, "step"), "log": getattr(distribution, "log"), } + return int_distribution if distribution.__class__.__name__ == "IntUniformDistribution": # Deprecated from Optuna v3.0 - return { + int_uniform: IntDistributionJSON = { "type": "IntDistribution", "low": getattr(distribution, "low"), "high": getattr(distribution, "high"), "step": getattr(distribution, "step"), "log": False, } + return int_uniform if distribution.__class__.__name__ == "IntLogUniformDistribution": # Deprecated from Optuna v3.0 - return { + int_log_uniform: IntDistributionJSON = { "type": "IntDistribution", "low": getattr(distribution, "low"), "high": getattr(distribution, "high"), "step": getattr(distribution, "step"), "log": True, } - + return int_log_uniform if isinstance(distribution, CategoricalDistribution): - return { + categorical: CategoricalDistributionJSON = { "type": "CategoricalDistribution", "choices": [ {"pytype": str(type(choice)), "value": str(choice)} for choice in distribution.choices ], } + return categorical raise ValueError(f"Unexpected distribution {str(distribution)}") From 734796df5249c05025a518fde5c05caebbac9d42 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 02:39:38 +0900 Subject: [PATCH 10/14] Fix a bug --- optuna_dashboard/_serializer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 1c26810c..7c50b6f2 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -150,7 +150,9 @@ def serialize_frozen_trial( ) -> dict[str, Any]: params = [] for param_name, param_external_value in trial.params.items(): - distribution = trial.distributions[param_name] + distribution = trial.distributions.get(param_name) + if distribution is None: + continue params.append( { "name": param_name, From 7d1bab030ec9e94e38f4be0eb266459a508fd72f Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 03:00:37 +0900 Subject: [PATCH 11/14] Use Target class on GraphEdf --- optuna_dashboard/ts/components/GraphEdf.tsx | 37 ++++++--------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphEdf.tsx b/optuna_dashboard/ts/components/GraphEdf.tsx index 6c706cc1..8e24f3ca 100644 --- a/optuna_dashboard/ts/components/GraphEdf.tsx +++ b/optuna_dashboard/ts/components/GraphEdf.tsx @@ -12,6 +12,7 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" +import {Target, useFilteredTrials, useObjectiveTargets} from "../trialFilter"; const plotDomId = "graph-edf" @@ -20,6 +21,8 @@ export const Edf: FC<{ }> = ({ study = null }) => { const theme = useTheme() const [objectiveId, setObjectiveId] = useState(0) + const targets = useObjectiveTargets(study) + const trials = useFilteredTrials(study, [targets[objectiveId]], false, false) const objectiveNames: string[] = study?.objective_names || [] const handleObjectiveChange = (event: SelectChangeEvent) => { @@ -28,9 +31,9 @@ export const Edf: FC<{ useEffect(() => { if (study != null) { - plotEdf(study, objectiveId, theme.palette.mode) + plotEdf(trials, targets[objectiveId], theme.palette.mode) } - }, [study, objectiveId, theme.palette.mode]) + }, [trials, targets, objectiveId, theme.palette.mode]) return ( Objective ID: @@ -65,24 +66,11 @@ export const Edf: FC<{ ) } -const filterFunc = (trial: Trial, objectiveId: number): boolean => { - return ( - trial.state === "Complete" && - trial.values !== undefined && - trial.values[objectiveId] !== "inf" && - trial.values[objectiveId] !== "-inf" - ) -} - -const plotEdf = (study: StudyDetail, objectiveId: number, mode: string) => { +const plotEdf = (trials: Trial[], target: Target, mode: string) => { if (document.getElementById(plotDomId) === null) { return } - - const trials: Trial[] = study ? study.trials : [] - const filteredTrials = trials.filter((t) => filterFunc(t, objectiveId)) - - if (filteredTrials.length === 0) { + if (trials.length === 0) { plotly.react(plotDomId, [], { template: mode === "dark" ? plotlyDarkTemplate : {}, }) @@ -90,11 +78,6 @@ const plotEdf = (study: StudyDetail, objectiveId: number, mode: string) => { } const target_name = "Objective Value" - - const target = (t: Trial): number => { - return t.values![objectiveId] as number - } - const layout: Partial = { xaxis: { title: target_name, @@ -111,7 +94,7 @@ const plotEdf = (study: StudyDetail, objectiveId: number, mode: string) => { template: mode === "dark" ? plotlyDarkTemplate : {}, } - const values = filteredTrials.map((t) => target(t)) + const values = trials.map((t) => target.getTargetValue(t) as number) const numValues = values.length const minX = Math.min(...values) const maxX = Math.max(...values) From e15317ee3e39ce7b34a053735e7e61bdeeb73d60 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 03:14:26 +0900 Subject: [PATCH 12/14] Remove redundant type conversion --- optuna_dashboard/ts/components/TrialTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/ts/components/TrialTable.tsx b/optuna_dashboard/ts/components/TrialTable.tsx index f3a55922..e88aebe6 100644 --- a/optuna_dashboard/ts/components/TrialTable.tsx +++ b/optuna_dashboard/ts/components/TrialTable.tsx @@ -160,7 +160,7 @@ export const TrialTable: FC<{ if (firstVal === secondVal) { return 0 } else if (firstVal && secondVal) { - return Number(firstVal) < Number(secondVal) ? 1 : -1 + return firstVal < secondVal ? 1 : -1 } else if (firstVal) { return -1 } else { From 491272ba05161769ca0354fc1f0f32f49adf3a85 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 03:53:45 +0900 Subject: [PATCH 13/14] Aggregate search space --- .../ts/components/GraphContour.tsx | 14 +------ optuna_dashboard/ts/components/GraphEdf.tsx | 2 +- optuna_dashboard/ts/searchSpace.ts | 37 +++++++++++++++++++ typescript_tests/searchSpace.test.ts | 34 +++++++++++++++++ 4 files changed, 74 insertions(+), 13 deletions(-) create mode 100644 optuna_dashboard/ts/searchSpace.ts create mode 100644 typescript_tests/searchSpace.test.ts diff --git a/optuna_dashboard/ts/components/GraphContour.tsx b/optuna_dashboard/ts/components/GraphContour.tsx index 06cb233b..dfa68e4e 100644 --- a/optuna_dashboard/ts/components/GraphContour.tsx +++ b/optuna_dashboard/ts/components/GraphContour.tsx @@ -1,5 +1,5 @@ import * as plotly from "plotly.js-dist-min" -import React, { FC, useEffect, useMemo, useState } from "react" +import React, { FC, useEffect, useState } from "react" import { Grid, FormControl, @@ -12,6 +12,7 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" +import { useSearchSpace } from "../searchSpace" // eslint-disable-next-line @typescript-eslint/no-explicit-any const unique = (array: any[]) => { @@ -33,17 +34,6 @@ type AxisInfo = { const PADDING_RATIO = 0.05 const plotDomId = "graph-contour" -const useSearchSpace = ( - unionSearchSpaces?: SearchSpaceItem[] -): SearchSpaceItem[] => - useMemo( - () => - Array.from(unionSearchSpaces || []).sort((a, b) => - a.name > b.name ? 1 : a.name < b.name ? -1 : 0 - ), - [unionSearchSpaces] - ) - export const Contour: FC<{ study: StudyDetail | null }> = ({ study = null }) => { diff --git a/optuna_dashboard/ts/components/GraphEdf.tsx b/optuna_dashboard/ts/components/GraphEdf.tsx index 8e24f3ca..81e73536 100644 --- a/optuna_dashboard/ts/components/GraphEdf.tsx +++ b/optuna_dashboard/ts/components/GraphEdf.tsx @@ -12,7 +12,7 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" -import {Target, useFilteredTrials, useObjectiveTargets} from "../trialFilter"; +import { Target, useFilteredTrials, useObjectiveTargets } from "../trialFilter" const plotDomId = "graph-edf" diff --git a/optuna_dashboard/ts/searchSpace.ts b/optuna_dashboard/ts/searchSpace.ts new file mode 100644 index 00000000..bd6fadef --- /dev/null +++ b/optuna_dashboard/ts/searchSpace.ts @@ -0,0 +1,37 @@ +import { useMemo } from "react" + +export const aggregateSearchSpace = ( + unionSearchSpace: SearchSpaceItem[] +): SearchSpaceItem[] => { + const knownElements = new Map() + unionSearchSpace.forEach((s) => { + const d = knownElements.get(s.name) + if (d === undefined) { + knownElements.set(s.name, s.distribution) + return + } + if ( + d.type === "CategoricalDistribution" || + s.distribution.type === "CategoricalDistribution" + ) { + // CategoricalDistribution.choices will never be changed + return + } + d.low = Math.min(d.low, s.distribution.low) + d.high = Math.max(d.low, s.distribution.high) + knownElements.set(s.name, d) + }) + return Array.from(knownElements.keys()) + .sort((a, b) => (a > b ? 1 : a < b ? -1 : 0)) + .map((name) => ({ + name: name, + distribution: knownElements.get(name) as Distribution, + })) +} + +export const useSearchSpace = ( + unionSearchSpaces?: SearchSpaceItem[] +): SearchSpaceItem[] => + useMemo(() => { + return aggregateSearchSpace(unionSearchSpaces || []) + }, [unionSearchSpaces]) diff --git a/typescript_tests/searchSpace.test.ts b/typescript_tests/searchSpace.test.ts new file mode 100644 index 00000000..1959d6c0 --- /dev/null +++ b/typescript_tests/searchSpace.test.ts @@ -0,0 +1,34 @@ +import { aggregateSearchSpace } from "../optuna_dashboard/ts/searchSpace" + +global.URL.createObjectURL = jest.fn() + +it("Aggregate SearchSpaceItem", () => { + const aggregated = aggregateSearchSpace([ + { + name: "float1", + distribution: { + type: "FloatDistribution", + low: 0, + high: 5, + step: 1, + log: true, + }, + }, + { + name: "float1", + distribution: { + type: "FloatDistribution", + low: 5, + high: 10, + step: 1, + log: true, + }, + }, + ]) + + expect(aggregated.length).toBe(1) + // @ts-ignore + expect(aggregated[0].distribution.low as number).toBe(0) + // @ts-ignore + expect(aggregated[0].distribution.high as number).toBe(10) +}) From 480d8b3d6c4303891270d1a634c86a37e0a183f4 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 7 Jan 2023 12:35:38 +0900 Subject: [PATCH 14/14] Fix GraphSlice --- .../ts/components/GraphContour.tsx | 16 +-- optuna_dashboard/ts/components/GraphEdf.tsx | 3 +- optuna_dashboard/ts/components/GraphSlice.tsx | 101 ++++++++---------- .../ts/components/PreferenceDialog.tsx | 9 ++ optuna_dashboard/ts/searchSpace.ts | 6 +- optuna_dashboard/ts/trialFilter.ts | 18 +++- typescript_tests/searchSpace.test.ts | 4 +- 7 files changed, 82 insertions(+), 75 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphContour.tsx b/optuna_dashboard/ts/components/GraphContour.tsx index dfa68e4e..11ccf617 100644 --- a/optuna_dashboard/ts/components/GraphContour.tsx +++ b/optuna_dashboard/ts/components/GraphContour.tsx @@ -12,7 +12,7 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" -import { useSearchSpace } from "../searchSpace" +import { useMergedUnionSearchSpace } from "../searchSpace" // eslint-disable-next-line @typescript-eslint/no-explicit-any const unique = (array: any[]) => { @@ -39,27 +39,27 @@ export const Contour: FC<{ }> = ({ study = null }) => { const theme = useTheme() const [objectiveId, setObjectiveId] = useState(0) - const searchSpaces = useSearchSpace(study?.union_search_space) + const searchSpace = useMergedUnionSearchSpace(study?.union_search_space) const [xParam, setXParam] = useState(null) const [yParam, setYParam] = useState(null) const objectiveNames: string[] = study?.objective_names || [] - if (xParam === null && searchSpaces.length > 0) { - setXParam(searchSpaces[0]) + if (xParam === null && searchSpace.length > 0) { + setXParam(searchSpace[0]) } - if (yParam === null && searchSpaces.length > 1) { - setYParam(searchSpaces[1]) + if (yParam === null && searchSpace.length > 1) { + setYParam(searchSpace[1]) } const handleObjectiveChange = (event: SelectChangeEvent) => { setObjectiveId(event.target.value as number) } const handleXParamChange = (event: SelectChangeEvent) => { - const param = searchSpaces.find((s) => s.name === event.target.value) + const param = searchSpace.find((s) => s.name === event.target.value) setXParam(param || null) } const handleYParamChange = (event: SelectChangeEvent) => { - const param = searchSpaces.find((s) => s.name === event.target.value) + const param = searchSpace.find((s) => s.name === event.target.value) setYParam(param || null) } diff --git a/optuna_dashboard/ts/components/GraphEdf.tsx b/optuna_dashboard/ts/components/GraphEdf.tsx index 81e73536..c0a32cf9 100644 --- a/optuna_dashboard/ts/components/GraphEdf.tsx +++ b/optuna_dashboard/ts/components/GraphEdf.tsx @@ -23,7 +23,6 @@ export const Edf: FC<{ const [objectiveId, setObjectiveId] = useState(0) const targets = useObjectiveTargets(study) const trials = useFilteredTrials(study, [targets[objectiveId]], false, false) - const objectiveNames: string[] = study?.objective_names || [] const handleObjectiveChange = (event: SelectChangeEvent) => { setObjectiveId(event.target.value as number) @@ -52,7 +51,7 @@ export const Edf: FC<{ diff --git a/optuna_dashboard/ts/components/GraphSlice.tsx b/optuna_dashboard/ts/components/GraphSlice.tsx index d1e201bf..4b8f8021 100644 --- a/optuna_dashboard/ts/components/GraphSlice.tsx +++ b/optuna_dashboard/ts/components/GraphSlice.tsx @@ -1,5 +1,5 @@ import * as plotly from "plotly.js-dist-min" -import React, { ChangeEvent, FC, useEffect, useMemo, useState } from "react" +import React, { ChangeEvent, FC, useEffect, useState } from "react" import { Grid, FormControl, @@ -13,22 +13,15 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" -import { Target, useFilteredTrials, useObjectiveTargets } from "../trialFilter" -import { useSnackbar } from "notistack" +import { + Target, + useFilteredTrials, + useObjectiveTargets, + useParamTargets, +} from "../trialFilter" const plotDomId = "graph-slice" -const useSearchSpace = ( - unionSearchSpaces?: SearchSpaceItem[] -): SearchSpaceItem[] => - useMemo( - () => - Array.from(unionSearchSpaces || []).sort((a, b) => - a.name > b.name ? 1 : a.name < b.name ? -1 : 0 - ), - [unionSearchSpaces] - ) - const isLogScale = (s: SearchSpaceItem): boolean => { if (s.distribution.type === "CategoricalDistribution") { return false @@ -40,49 +33,43 @@ export const GraphSlice: FC<{ study: StudyDetail | null }> = ({ study = null }) => { const theme = useTheme() - const { enqueueSnackbar } = useSnackbar() const [objectiveId, setObjectiveId] = useState(0) - const [selected, setSelected] = useState(null) + const objectiveTargets = useObjectiveTargets(study) + const [paramTargetsIndex, setParamTargetsIndex] = useState(0) + const [paramTargets, searchSpace] = useParamTargets(study) const [logYScale, setLogYScale] = useState(false) - const searchSpaces = useSearchSpace(study?.union_search_space) - const targets = useObjectiveTargets(study) - const filterTargets: Target[] = [targets[objectiveId]] - if (selected !== null) filterTargets.push(new Target("params", selected.name)) + const filterTargets: Target[] = [objectiveTargets[objectiveId]] + if (paramTargets.length > paramTargetsIndex) + filterTargets.push(paramTargets[paramTargetsIndex]) const trials = useFilteredTrials(study, filterTargets, false, false) - const objectiveNames: string[] = study?.objective_names || [] - if (selected === null && searchSpaces.length > 0) { - setSelected(searchSpaces[0]) - } - useEffect(() => { plotSlice( trials, - targets[objectiveId], - selected, + objectiveTargets[objectiveId], + searchSpace.length > paramTargetsIndex + ? searchSpace[paramTargetsIndex] + : null, logYScale, theme.palette.mode ) - }, [trials, targets[objectiveId], selected, logYScale, theme.palette.mode]) + }, [ + trials, + objectiveTargets[objectiveId], + searchSpace, + paramTargetsIndex, + logYScale, + theme.palette.mode, + ]) const handleObjectiveChange = (event: SelectChangeEvent) => { setObjectiveId(event.target.value as number) } - const handleSelectedParam = (e: SelectChangeEvent) => { - const s = searchSpaces.find((s) => s.name === e.target.value) - if (s === undefined) { - enqueueSnackbar( - `Cannot find ${e.target.value} param in the search space.`, - { - variant: "error", - } - ) - return - } - setSelected(s) + const handleSelectedParam = (e: SelectChangeEvent) => { + setParamTargetsIndex(e.target.value as number) } const handleLogYScaleChange = (e: ChangeEvent) => { @@ -105,28 +92,26 @@ export const GraphSlice: FC<{ Objective ID: + + )} + {paramTargets.length !== 0 && paramTargetsIndex !== null && ( + + Parameter: + )} - - Parameter: - - Log y scale: target.getTargetValue(t) as number + (t) => objectiveTarget.getTargetValue(t) as number ) const paramTarget = new Target("params", selected.name) const values = trials.map((t) => paramTarget.getTargetValue(t) as number) diff --git a/optuna_dashboard/ts/components/PreferenceDialog.tsx b/optuna_dashboard/ts/components/PreferenceDialog.tsx index 79261f7f..399144e8 100644 --- a/optuna_dashboard/ts/components/PreferenceDialog.tsx +++ b/optuna_dashboard/ts/components/PreferenceDialog.tsx @@ -49,6 +49,15 @@ export const usePreferenceDialog = ( [event.target.name]: event.target.checked, }) } + const renderSelectBox = (onChange: (e) => void): ReactNode => ( + + ) const renderPreferenceDialog = () => { return ( diff --git a/optuna_dashboard/ts/searchSpace.ts b/optuna_dashboard/ts/searchSpace.ts index bd6fadef..729f1f51 100644 --- a/optuna_dashboard/ts/searchSpace.ts +++ b/optuna_dashboard/ts/searchSpace.ts @@ -1,6 +1,6 @@ import { useMemo } from "react" -export const aggregateSearchSpace = ( +export const mergeUnionSearchSpace = ( unionSearchSpace: SearchSpaceItem[] ): SearchSpaceItem[] => { const knownElements = new Map() @@ -29,9 +29,9 @@ export const aggregateSearchSpace = ( })) } -export const useSearchSpace = ( +export const useMergedUnionSearchSpace = ( unionSearchSpaces?: SearchSpaceItem[] ): SearchSpaceItem[] => useMemo(() => { - return aggregateSearchSpace(unionSearchSpaces || []) + return mergeUnionSearchSpace(unionSearchSpaces || []) }, [unionSearchSpaces]) diff --git a/optuna_dashboard/ts/trialFilter.ts b/optuna_dashboard/ts/trialFilter.ts index 1e249761..204fa817 100644 --- a/optuna_dashboard/ts/trialFilter.ts +++ b/optuna_dashboard/ts/trialFilter.ts @@ -1,4 +1,5 @@ import { useMemo } from "react" +import { mergeUnionSearchSpace } from "./searchSpace" type TargetKind = "objective" | "user_attr" | "params" @@ -28,10 +29,10 @@ export class Target { return true } - toLabel(objectiveNames: string[]): string { + toLabel(objectiveNames?: string[]): string { if (this.kind === "objective") { const objectiveId: number = this.key as number - if (objectiveNames.length > objectiveId) { + if (objectiveNames !== undefined && objectiveNames.length > objectiveId) { return objectiveNames[objectiveId] } return `Objective ${objectiveId}` @@ -121,6 +122,19 @@ export const useObjectiveTargets = (study: StudyDetail | null): Target[] => } }, [study?.directions]) +export const useParamTargets = ( + study: StudyDetail | null +): [Target[], SearchSpaceItem[]] => + useMemo<[Target[], SearchSpaceItem[]]>(() => { + if (study !== null) { + const searchSpace = mergeUnionSearchSpace(study.union_search_space) + const targets = searchSpace.map((s) => new Target("params", s.name)) + return [targets, searchSpace] + } else { + return [[], []] + } + }, [study?.union_search_space]) + export const useObjectiveAndSystemAttrTargets = ( study: StudyDetail | null ): Target[] => diff --git a/typescript_tests/searchSpace.test.ts b/typescript_tests/searchSpace.test.ts index 1959d6c0..fa36be7c 100644 --- a/typescript_tests/searchSpace.test.ts +++ b/typescript_tests/searchSpace.test.ts @@ -1,9 +1,9 @@ -import { aggregateSearchSpace } from "../optuna_dashboard/ts/searchSpace" +import { mergeUnionSearchSpace } from "../optuna_dashboard/ts/searchSpace" global.URL.createObjectURL = jest.fn() it("Aggregate SearchSpaceItem", () => { - const aggregated = aggregateSearchSpace([ + const aggregated = mergeUnionSearchSpace([ { name: "float1", distribution: {