diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 6442fa91..7c50b6f2 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -7,6 +7,7 @@ from typing import Union import numpy as np from optuna.distributions import BaseDistribution +from optuna.distributions import CategoricalDistribution from optuna.study import StudySummary from optuna.trial import FrozenTrial @@ -40,6 +41,44 @@ if TYPE_CHECKING: }, ) + FloatDistributionJSON = TypedDict( + "FloatDistributionJSON", + { + "type": Literal["FloatDistribution"], + "low": float, + "high": float, + "step": float, + "log": bool, + }, + ) + IntDistributionJSON = TypedDict( + "IntDistributionJSON", + { + "type": Literal["IntDistribution"], + "low": int, + "high": int, + "step": int, + "log": bool, + }, + ) + CategoricalDistributionChoiceJSON = TypedDict( + "CategoricalDistributionChoiceJSON", + { + "pytype": str, + "value": str, + }, + ) + CategoricalDistributionJSON = TypedDict( + "CategoricalDistributionJSON", + { + "type": Literal["CategoricalDistribution"], + "choices": list[CategoricalDistributionChoiceJSON], + }, + ) + DistributionJSON = Union[ + FloatDistributionJSON, IntDistributionJSON, CategoricalDistributionJSON + ] + MAX_ATTR_LENGTH = 1024 @@ -109,12 +148,26 @@ def serialize_study_detail( def serialize_frozen_trial( study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any] ) -> dict[str, Any]: + params = [] + for param_name, param_external_value in trial.params.items(): + distribution = trial.distributions.get(param_name) + if distribution is None: + continue + params.append( + { + "name": param_name, + "param_internal_value": distribution.to_internal_repr(param_external_value), + "param_external_value": str(param_external_value), + "param_external_pytyp": str(type(param_external_value)), + "distribution": serialize_distribution(distribution), + } + ) serialized = { "trial_id": trial._trial_id, "study_id": study_id, "number": trial.number, "state": trial.state.name.capitalize(), - "params": [{"name": name, "value": str(value)} for name, value in trial.params.items()], + "params": params, "user_attrs": serialize_attrs(trial.user_attrs), "system_attrs": serialize_attrs(getattr(trial, "_system_attrs", {})), "note": note.get_note_from_system_attrs(study_system_attrs, trial._trial_id), @@ -158,6 +211,89 @@ def serialize_frozen_trial( return serialized +def serialize_distribution(distribution: BaseDistribution) -> DistributionJSON: + if distribution.__class__.__name__ == "FloatDistribution": + # Added from Optuna v3.0 + float_distribution: FloatDistributionJSON = { + "type": "FloatDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "step"), + "log": getattr(distribution, "log"), + } + return float_distribution + if distribution.__class__.__name__ == "UniformDistribution": + # Deprecated from Optuna v3.0 + uniform: FloatDistributionJSON = { + "type": "FloatDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": 0, + "log": False, + } + return uniform + if distribution.__class__.__name__ == "LogUniformDistribution": + # Deprecated from Optuna v3.0 + log_uniform: FloatDistributionJSON = { + "type": "FloatDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": 0, + "log": True, + } + return log_uniform + if distribution.__class__.__name__ == "DiscreteUniformDistribution": + # Deprecated from Optuna v3.0 + discrete_uniform: FloatDistributionJSON = { + "type": "FloatDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "q"), + "log": False, + } + return discrete_uniform + if distribution.__class__.__name__ == "IntDistribution": + # Added from Optuna v3.0 + int_distribution: IntDistributionJSON = { + "type": "IntDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "step"), + "log": getattr(distribution, "log"), + } + return int_distribution + if distribution.__class__.__name__ == "IntUniformDistribution": + # Deprecated from Optuna v3.0 + int_uniform: IntDistributionJSON = { + "type": "IntDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "step"), + "log": False, + } + return int_uniform + if distribution.__class__.__name__ == "IntLogUniformDistribution": + # Deprecated from Optuna v3.0 + int_log_uniform: IntDistributionJSON = { + "type": "IntDistribution", + "low": getattr(distribution, "low"), + "high": getattr(distribution, "high"), + "step": getattr(distribution, "step"), + "log": True, + } + return int_log_uniform + if isinstance(distribution, CategoricalDistribution): + categorical: CategoricalDistributionJSON = { + "type": "CategoricalDistribution", + "choices": [ + {"pytype": str(type(choice)), "value": str(choice)} + for choice in distribution.choices + ], + } + return categorical + raise ValueError(f"Unexpected distribution {str(distribution)}") + + def serialize_search_space( search_space: list[tuple[str, BaseDistribution]] ) -> list[dict[str, Any]]: @@ -166,8 +302,7 @@ def serialize_search_space( serialized.append( { "name": param_name, - "distribution": distribution.__class__.__name__, - "attributes": distribution._asdict(), + "distribution": serialize_distribution(distribution), } ) return serialized diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index f98cbd18..5b3fc2fe 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -44,8 +44,8 @@ interface StudyDetailResponse { directions: StudyDirection[] trials: TrialResponse[] best_trials: TrialResponse[] - intersection_search_space: SearchSpace[] - union_search_space: SearchSpace[] + intersection_search_space: SearchSpaceItem[] + union_search_space: SearchSpaceItem[] union_user_attrs: AttributeSpec[] has_intermediate_values: boolean note: Note diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index bc889c7e..99306435 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -33,6 +33,13 @@ import { Switch } from "@mui/material" const drawerWidth = 240 +export type PageId = + | "history" + | "analytics" + | "trialTable" + | "trialList" + | "note" + const openedMixin = (theme: Theme): CSSObject => ({ width: drawerWidth, transition: theme.transitions.create("width", { diff --git a/optuna_dashboard/ts/components/BestTrialsCard.tsx b/optuna_dashboard/ts/components/BestTrialsCard.tsx index c1a5d7b4..d34007e5 100644 --- a/optuna_dashboard/ts/components/BestTrialsCard.tsx +++ b/optuna_dashboard/ts/components/BestTrialsCard.tsx @@ -41,7 +41,10 @@ export const BestTrialsCard: FC<{ Params = [ - {bestTrial.params.map((p) => `${p.name}: ${p.value}`).join(", ")}] + {bestTrial.params + .map((p) => `${p.name}: ${p.param_external_value}`) + .join(", ")} + ] Intermediate Values = [ @@ -88,26 +91,23 @@ export const BestTrialsCard: FC<{ URL_PREFIX + `/studies/${trial.study_id}/trials?numbers=${trial.number}` } + sx={{ flexDirection: "column", alignItems: "flex-start" }} > Trial {trial.number} } - secondary={ - <> - - Objective Values = [{trial.values?.join(", ")}] - - - Params = [ - {trial.params - .map((p) => `${p.name}: ${p.value}`) - .join(", ")} - ] - - - } /> + + Objective Values = [{trial.values?.join(", ")}] + + + Params = [ + {trial.params + .map((p) => `${p.name}: ${p.param_external_value}`) + .join(", ")} + ] + ))} diff --git a/optuna_dashboard/ts/components/GraphContour.tsx b/optuna_dashboard/ts/components/GraphContour.tsx index b8301e20..11ccf617 100644 --- a/optuna_dashboard/ts/components/GraphContour.tsx +++ b/optuna_dashboard/ts/components/GraphContour.tsx @@ -12,6 +12,7 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" +import { useMergedUnionSearchSpace } from "../searchSpace" // eslint-disable-next-line @typescript-eslint/no-explicit-any const unique = (array: any[]) => { @@ -38,26 +39,28 @@ export const Contour: FC<{ }> = ({ study = null }) => { const theme = useTheme() const [objectiveId, setObjectiveId] = useState(0) - const [xParam, setXParam] = useState("") - const [yParam, setYParam] = useState("") - const paramNames = study?.union_search_space.map((s) => s.name) + const searchSpace = useMergedUnionSearchSpace(study?.union_search_space) + const [xParam, setXParam] = useState(null) + const [yParam, setYParam] = useState(null) const objectiveNames: string[] = study?.objective_names || [] - if (!xParam && paramNames && paramNames.length > 0) { - setXParam(paramNames[0]) + if (xParam === null && searchSpace.length > 0) { + setXParam(searchSpace[0]) } - if (!yParam && paramNames && paramNames.length > 1) { - setYParam(paramNames[1]) + if (yParam === null && searchSpace.length > 1) { + setYParam(searchSpace[1]) } const handleObjectiveChange = (event: SelectChangeEvent) => { setObjectiveId(event.target.value as number) } const handleXParamChange = (event: SelectChangeEvent) => { - setXParam(event.target.value as string) + const param = searchSpace.find((s) => s.name === event.target.value) + setXParam(param || null) } const handleYParamChange = (event: SelectChangeEvent) => { - setYParam(event.target.value as string) + const param = searchSpace.find((s) => s.name === event.target.value) + setYParam(param || null) } useEffect(() => { @@ -66,7 +69,7 @@ export const Contour: FC<{ } }, [study, objectiveId, xParam, yParam, theme.palette.mode]) - const space: SearchSpace[] = study ? study.union_search_space : [] + const space: SearchSpaceItem[] = study ? study.union_search_space : [] return ( @@ -98,7 +101,7 @@ export const Contour: FC<{ x: - {space.map((d, i) => ( {d.name} @@ -108,7 +111,7 @@ export const Contour: FC<{ y: - {space.map((d, i) => ( {d.name} @@ -126,73 +129,6 @@ export const Contour: FC<{ ) } -const isNumerical = (trials: Trial[], paramName: string): boolean => { - return trials.every((t) => { - const param = t.params.find((param) => param.name === paramName) - if (!param) return true - const val = param.value - return typeof (Number(val) || val) === "number" - }) -} - -const getAxisInfo = (trials: Trial[], paramName: string): AxisInfo => { - const values = trials.map((trial) => { - const param = trial.params.find((p) => p.name === paramName) - return param ? Number(param.value) || param.value : null - }) - - let min: number - let max: number - let isLog: boolean - let isCat: boolean - - if (isNumerical(trials, paramName)) { - const minValue = Math.min(...(values as number[])) - const maxValue = Math.max(...(values as number[])) - const padding = (maxValue - minValue) * PADDING_RATIO - min = minValue - padding - max = maxValue + padding - isLog = false - isCat = false - } else { - const uniqueValues = unique(values) - const span = uniqueValues.length - (uniqueValues.includes(null) ? 2 : 1) - const padding = span * PADDING_RATIO - min = -padding - max = span + padding - isLog = false - isCat = true - } - - const indices = isNumerical(trials, paramName) - ? unique((values as (number | null)[]).filter((v) => v !== null)).sort( - (a, b) => a - b - ) - : unique((values as (string | null)[]).filter((v) => v !== null)).sort( - (a, b) => - a.toString().toLowerCase() < b.toString().toLowerCase() - ? -1 - : a.toString().toLowerCase() > b.toString().toLowerCase() - ? 1 - : 0 - ) - - if (indices.length >= 2 && isNumerical(trials, paramName)) { - indices.unshift(min) - indices.push(max) - } - - return { - name: paramName, - min, - max, - isLog, - isCat, - indices, - values, - } -} - const filterFunc = (trial: Trial, objectiveId: number): boolean => { return ( trial.state === "Complete" && @@ -205,8 +141,8 @@ const filterFunc = (trial: Trial, objectiveId: number): boolean => { const plotContour = ( study: StudyDetail, objectiveId: number, - xParam: string, - yParam: string, + xParam: SearchSpaceItem | null, + yParam: SearchSpaceItem | null, mode: string ) => { if (document.getElementById(plotDomId) === null) { @@ -215,16 +151,15 @@ const plotContour = ( const trials: Trial[] = study ? study.trials : [] const filteredTrials = trials.filter((t) => filterFunc(t, objectiveId)) - - if (filteredTrials.length === 0) { + if (filteredTrials.length === 0 || xParam === null || yParam === null) { plotly.react(plotDomId, [], { template: mode === "dark" ? plotlyDarkTemplate : {}, }) return } - const xAxis = getAxisInfo(trials, xParam) - const yAxis = getAxisInfo(trials, yParam) + const xAxis = getAxisInfo(study, trials, xParam) + const yAxis = getAxisInfo(study, trials, yParam) const xIndices = xAxis.indices const yIndices = yAxis.indices @@ -279,11 +214,11 @@ const plotContour = ( const layout: Partial = { xaxis: { - title: xParam, + title: xParam.name, type: xAxis.isCat ? "category" : undefined, }, yaxis: { - title: yParam, + title: yParam.name, type: yAxis.isCat ? "category" : undefined, }, margin: { @@ -296,3 +231,87 @@ const plotContour = ( } plotly.react(plotDomId, plotData, layout) } + +const getAxisInfoForNumericalParams = ( + trials: Trial[], + paramName: string, + distribution: FloatDistribution | IntDistribution +): AxisInfo => { + const padding = (distribution.high - distribution.low) * PADDING_RATIO + const min = distribution.low - padding + const max = distribution.high + padding + + const values = trials.map( + (trial) => + trial.params.find((p) => p.name === paramName)?.param_internal_value || + null + ) + const indices = unique(values) + .filter((v) => v !== null) + .sort((a, b) => a - b) + if (indices.length >= 2) { + indices.unshift(min) + indices.push(max) + } + return { + name: paramName, + min, + max, + isLog: distribution.log, + isCat: false, + indices, + values, + } +} + +const getAxisInfoForCategoricalParams = ( + trials: Trial[], + paramName: string, + distribution: CategoricalDistribution +): AxisInfo => { + const values = trials.map( + (trial) => + trial.params.find((p) => p.name === paramName)?.param_external_value || + null + ) + const isDynamic = values.some((v) => v === null) + const span = distribution.choices.length - (isDynamic ? 2 : 1) + const padding = span * PADDING_RATIO + const min = -padding + const max = span + padding + + const indices = distribution.choices + .map((c) => c.value) + .sort((a, b) => + a.toLowerCase() < b.toLowerCase() + ? -1 + : a.toLowerCase() > b.toLowerCase() + ? 1 + : 0 + ) + return { + name: paramName, + min, + max, + isLog: false, + isCat: true, + indices, + values, + } +} + +const getAxisInfo = ( + study: StudyDetail, + trials: Trial[], + param: SearchSpaceItem +): AxisInfo => { + if (param.distribution.type === "CategoricalDistribution") { + return getAxisInfoForCategoricalParams( + trials, + param.name, + param.distribution + ) + } else { + return getAxisInfoForNumericalParams(trials, param.name, param.distribution) + } +} diff --git a/optuna_dashboard/ts/components/GraphEdf.tsx b/optuna_dashboard/ts/components/GraphEdf.tsx index 6c706cc1..c0a32cf9 100644 --- a/optuna_dashboard/ts/components/GraphEdf.tsx +++ b/optuna_dashboard/ts/components/GraphEdf.tsx @@ -12,6 +12,7 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" +import { Target, useFilteredTrials, useObjectiveTargets } from "../trialFilter" const plotDomId = "graph-edf" @@ -20,7 +21,8 @@ export const Edf: FC<{ }> = ({ study = null }) => { const theme = useTheme() const [objectiveId, setObjectiveId] = useState(0) - const objectiveNames: string[] = study?.objective_names || [] + const targets = useObjectiveTargets(study) + const trials = useFilteredTrials(study, [targets[objectiveId]], false, false) const handleObjectiveChange = (event: SelectChangeEvent) => { setObjectiveId(event.target.value as number) @@ -28,9 +30,9 @@ export const Edf: FC<{ useEffect(() => { if (study != null) { - plotEdf(study, objectiveId, theme.palette.mode) + plotEdf(trials, targets[objectiveId], theme.palette.mode) } - }, [study, objectiveId, theme.palette.mode]) + }, [trials, targets, objectiveId, theme.palette.mode]) return ( Objective ID: @@ -65,24 +65,11 @@ export const Edf: FC<{ ) } -const filterFunc = (trial: Trial, objectiveId: number): boolean => { - return ( - trial.state === "Complete" && - trial.values !== undefined && - trial.values[objectiveId] !== "inf" && - trial.values[objectiveId] !== "-inf" - ) -} - -const plotEdf = (study: StudyDetail, objectiveId: number, mode: string) => { +const plotEdf = (trials: Trial[], target: Target, mode: string) => { if (document.getElementById(plotDomId) === null) { return } - - const trials: Trial[] = study ? study.trials : [] - const filteredTrials = trials.filter((t) => filterFunc(t, objectiveId)) - - if (filteredTrials.length === 0) { + if (trials.length === 0) { plotly.react(plotDomId, [], { template: mode === "dark" ? plotlyDarkTemplate : {}, }) @@ -90,11 +77,6 @@ const plotEdf = (study: StudyDetail, objectiveId: number, mode: string) => { } const target_name = "Objective Value" - - const target = (t: Trial): number => { - return t.values![objectiveId] as number - } - const layout: Partial = { xaxis: { title: target_name, @@ -111,7 +93,7 @@ const plotEdf = (study: StudyDetail, objectiveId: number, mode: string) => { template: mode === "dark" ? plotlyDarkTemplate : {}, } - const values = filteredTrials.map((t) => target(t)) + const values = trials.map((t) => target.getTargetValue(t) as number) const numValues = values.length const minX = Math.min(...values) const maxX = Math.max(...values) diff --git a/optuna_dashboard/ts/components/GraphHistory.tsx b/optuna_dashboard/ts/components/GraphHistory.tsx index b7c13940..8c38fc48 100644 --- a/optuna_dashboard/ts/components/GraphHistory.tsx +++ b/optuna_dashboard/ts/components/GraphHistory.tsx @@ -1,5 +1,5 @@ import * as plotly from "plotly.js-dist-min" -import React, { ChangeEvent, FC, useEffect, useMemo, useState } from "react" +import React, { ChangeEvent, FC, useEffect, useState } from "react" import { Grid, FormControl, @@ -16,120 +16,47 @@ import { useTheme, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" +import { + useFilteredTrials, + Target, + useObjectiveAndSystemAttrTargets, +} from "../trialFilter" const plotDomId = "graph-history" -class Target { - kind: "objective" | "user_attr" - key: number | string - - constructor(kind: "objective" | "user_attr", key: number | string) { - this.kind = kind - this.key = key - } - - validate(): boolean { - if (this.kind === "objective") { - if (typeof this.key !== "number") { - return false - } - } else if (this.kind === "user_attr") { - if (typeof this.key !== "string") { - return false - } - } else { - return false - } - return true - } - - toLabel(objectiveNames: string[]): string { - if (this.kind === "objective") { - const objectiveId: number = this.key as number - if (objectiveNames.length > objectiveId) { - return objectiveNames[objectiveId] - } - return `Objective ${objectiveId}` - } else { - return `User Attribute ${this.key}` - } - } - - getObjectiveId(): number | null { - return this.key as number - } - - getTargetValue(trial: Trial): number | null { - if (!this.validate()) { - return null - } - if (this.kind === "objective") { - const objectiveId = this.getObjectiveId() - if ( - objectiveId === null || - trial.values === undefined || - trial.values.length <= objectiveId - ) { - return null - } - const value = trial.values[objectiveId] - if (value === "inf" || value === "-inf") { - return null - } - return value - } else if (this.kind === "user_attr") { - const attr = trial.user_attrs.find((attr) => attr.key === this.key) - if (attr === undefined) { - return null - } - const value = Number(attr.value) - if (value === undefined) { - return null - } - return value - } - return null - } -} - export const GraphHistory: FC<{ study: StudyDetail | null }> = ({ study = null }) => { const theme = useTheme() const [xAxis, setXAxis] = useState("number") - const [targetIndex, setTargetIndex] = useState(0) const [logScale, setLogScale] = useState(false) const [filterCompleteTrial, setFilterCompleteTrial] = useState(false) const [filterPrunedTrial, setFilterPrunedTrial] = useState(false) - const [targetList, setTargetList] = useState([]) - const objectiveNames: string[] = study?.objective_names || [] - useMemo(() => { - if (study !== null) { - const targets: Target[] = [ - ...study.directions.map((v, i) => new Target("objective", i)), - ...study.union_user_attrs - .filter((attr) => attr.sortable) - .map((attr) => new Target("user_attr", attr.key)), - ] - setTargetList(targets) - } - }, [study?.directions, study?.union_user_attrs]) + const objectiveNames: string[] = study?.objective_names || [] + const targetList = useObjectiveAndSystemAttrTargets(study) + const [targetIndex, setTargetIndex] = useState(0) + const trials = useFilteredTrials( + study, + [targetList[targetIndex]], + filterCompleteTrial, + filterPrunedTrial + ) useEffect(() => { if (study !== null) { plotHistory( - study, + trials, + study.directions, targetList[targetIndex], xAxis, logScale, - filterCompleteTrial, - filterPrunedTrial, theme.palette.mode ) } }, [ - study, + trials, + study?.directions, targetIndex, targetList, logScale, @@ -258,21 +185,12 @@ export const GraphHistory: FC<{ ) } -const filterFunc = (trial: Trial, target: Target): boolean => { - if (trial.state !== "Complete" && trial.state !== "Pruned") { - return false - } - const value = target.getTargetValue(trial) - return value !== null -} - const plotHistory = ( - study: StudyDetail, + trials: Trial[], + directions: StudyDirection[], target: Target, xAxis: string, logScale: boolean, - filterCompleteTrial: boolean, - filterPrunedTrial: boolean, mode: string ) => { if (document.getElementById(plotDomId) === null) { @@ -297,15 +215,7 @@ const plotHistory = ( showlegend: true, template: mode === "dark" ? plotlyDarkTemplate : {}, } - - let filteredTrials = study.trials.filter((t) => filterFunc(t, target)) - if (filterCompleteTrial) { - filteredTrials = filteredTrials.filter((t) => t.state !== "Complete") - } - if (filterPrunedTrial) { - filteredTrials = filteredTrials.filter((t) => t.state !== "Pruned") - } - if (filteredTrials.length === 0) { + if (trials.length === 0) { plotly.react(plotDomId, [], layout) return } @@ -320,10 +230,8 @@ const plotHistory = ( const plotData: Partial[] = [ { - x: filteredTrials.map(getAxisX), - y: filteredTrials.map( - (t: Trial): number => target.getTargetValue(t) as number - ), + x: trials.map(getAxisX), + y: trials.map((t: Trial): number => target.getTargetValue(t) as number), name: "Objective Value", mode: "markers", type: "scatter", @@ -335,17 +243,17 @@ const plotHistory = ( const xForLinePlot: (number | Date)[] = [] const yForLinePlot: number[] = [] let currentBest: number | null = null - for (let i = 0; i < filteredTrials.length; i++) { - const t = filteredTrials[i] + for (let i = 0; i < trials.length; i++) { + const t = trials[i] if (currentBest === null) { currentBest = t.values![objectiveId] as number xForLinePlot.push(getAxisX(t)) yForLinePlot.push(t.values![objectiveId] as number) } else if ( - study.directions[objectiveId] === "maximize" && + directions[objectiveId] === "maximize" && t.values![objectiveId] > currentBest ) { - const p = filteredTrials[i - 1] + const p = trials[i - 1] if (!xForLinePlot.includes(getAxisX(p))) { xForLinePlot.push(getAxisX(p)) yForLinePlot.push(currentBest) @@ -354,10 +262,10 @@ const plotHistory = ( xForLinePlot.push(getAxisX(t)) yForLinePlot.push(t.values![objectiveId] as number) } else if ( - study.directions[objectiveId] === "minimize" && + directions[objectiveId] === "minimize" && t.values![objectiveId] < currentBest ) { - const p = filteredTrials[i - 1] + const p = trials[i - 1] if (!xForLinePlot.includes(getAxisX(p))) { xForLinePlot.push(getAxisX(p)) yForLinePlot.push(currentBest) @@ -367,7 +275,7 @@ const plotHistory = ( yForLinePlot.push(t.values![objectiveId] as number) } } - xForLinePlot.push(getAxisX(filteredTrials[filteredTrials.length - 1])) + xForLinePlot.push(getAxisX(trials[trials.length - 1])) yForLinePlot.push(yForLinePlot[yForLinePlot.length - 1]) plotData.push({ x: xForLinePlot, diff --git a/optuna_dashboard/ts/components/GraphParallelCoordinate.tsx b/optuna_dashboard/ts/components/GraphParallelCoordinate.tsx index ca394e62..7347f42e 100644 --- a/optuna_dashboard/ts/components/GraphParallelCoordinate.tsx +++ b/optuna_dashboard/ts/components/GraphParallelCoordinate.tsx @@ -136,15 +136,10 @@ const plotCoordinate = ( }, ] study.intersection_search_space.forEach((s) => { - const valueStrings = filteredTrials.map((t) => { - const param = t.params.find((p) => p.name === s.name) - return param!.value - }) - const isnum = valueStrings.every((v) => { - return !isNaN(Number(v)) - }) - if (isnum) { - const values: number[] = valueStrings.map((v) => parseFloat(v)) + const values: number[] = filteredTrials.map( + (t) => t.params.find((p) => p.name === s.name)!.param_internal_value + ) + if (s.distribution.type !== "CategoricalDistribution") { dimensions.push({ label: breakLabelIfTooLong(s.name), values: values, @@ -152,11 +147,7 @@ const plotCoordinate = ( }) } else { // categorical - const vocabSet = new Set(valueStrings) - const vocabArr = Array.from(vocabSet) - const values: number[] = valueStrings.map((v) => - vocabArr.findIndex((vocab) => v === vocab) - ) + const vocabArr: string[] = s.distribution.choices.map((c) => c.value) const tickvals: number[] = vocabArr.map((v, i) => i) dimensions.push({ label: breakLabelIfTooLong(s.name), diff --git a/optuna_dashboard/ts/components/GraphParetoFront.tsx b/optuna_dashboard/ts/components/GraphParetoFront.tsx index ef3dd646..22c4007c 100644 --- a/optuna_dashboard/ts/components/GraphParetoFront.tsx +++ b/optuna_dashboard/ts/components/GraphParetoFront.tsx @@ -105,7 +105,7 @@ const makeHovertext = (trial: Trial): string => { number: trial.number, values: trial.values, params: trial.params - .map((p) => [p.name, p.value]) + .map((p) => [p.name, p.param_external_value]) .reduce((obj, [key, value]) => ({ ...obj, [key]: value }), {}), }, undefined, diff --git a/optuna_dashboard/ts/components/GraphSlice.tsx b/optuna_dashboard/ts/components/GraphSlice.tsx index 31231881..4b8f8021 100644 --- a/optuna_dashboard/ts/components/GraphSlice.tsx +++ b/optuna_dashboard/ts/components/GraphSlice.tsx @@ -13,52 +13,63 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" +import { + Target, + useFilteredTrials, + useObjectiveTargets, + useParamTargets, +} from "../trialFilter" const plotDomId = "graph-slice" -// TODO(c-bata): Check `log` field of IntDistribution and FloatDistribution. -const logDistributions = ["LogUniformDistribution", "IntLogUniformDistribution"] +const isLogScale = (s: SearchSpaceItem): boolean => { + if (s.distribution.type === "CategoricalDistribution") { + return false + } + return s.distribution.log +} export const GraphSlice: FC<{ study: StudyDetail | null }> = ({ study = null }) => { const theme = useTheme() - const trials: Trial[] = study !== null ? study.trials : [] + const [objectiveId, setObjectiveId] = useState(0) - const [selected, setSelected] = useState(null) - const [logXScale, setLogXScale] = useState(false) + const objectiveTargets = useObjectiveTargets(study) + const [paramTargetsIndex, setParamTargetsIndex] = useState(0) + const [paramTargets, searchSpace] = useParamTargets(study) const [logYScale, setLogYScale] = useState(false) - const paramNames = study?.union_search_space.map((s) => s.name) - const distributions = new Map( - study?.union_search_space.map((s) => [s.name, s.distribution]) - ) - const objectiveNames: string[] = study?.objective_names || [] - if (selected === null && paramNames && paramNames.length > 0) { - const distribution = distributions.get(paramNames[0]) || "" - setSelected(paramNames[0]) - setLogXScale(logDistributions.includes(distribution)) - } + + const filterTargets: Target[] = [objectiveTargets[objectiveId]] + if (paramTargets.length > paramTargetsIndex) + filterTargets.push(paramTargets[paramTargetsIndex]) + const trials = useFilteredTrials(study, filterTargets, false, false) useEffect(() => { plotSlice( trials, - objectiveId, - selected, - logXScale, + objectiveTargets[objectiveId], + searchSpace.length > paramTargetsIndex + ? searchSpace[paramTargetsIndex] + : null, logYScale, theme.palette.mode ) - }, [trials, objectiveId, selected, logXScale, logYScale, theme.palette.mode]) + }, [ + trials, + objectiveTargets[objectiveId], + searchSpace, + paramTargetsIndex, + logYScale, + theme.palette.mode, + ]) const handleObjectiveChange = (event: SelectChangeEvent) => { setObjectiveId(event.target.value as number) } - const handleSelectedParam = (e: SelectChangeEvent) => { - const paramName = e.target.value - const distribution = distributions.get(paramName) || "" - setSelected(paramName) - setLogXScale(logDistributions.includes(distribution)) + const handleSelectedParam = (e: SelectChangeEvent) => { + setParamTargetsIndex(e.target.value as number) } const handleLogYScaleChange = (e: ChangeEvent) => { @@ -81,28 +92,26 @@ export const GraphSlice: FC<{ Objective ID: + + )} + {paramTargets.length !== 0 && paramTargetsIndex !== null && ( + + Parameter: + )} - - Parameter: - - Log y scale: { - if (trial.state !== "Complete" && trial.state !== "Pruned") { - return false - } - if (trial.params.find((p) => p.name == selected) === undefined) { - return false - } - if (trial.values === undefined) { - return false - } - return ( - trial.values.length > objectiveId && - trial.values[objectiveId] !== "inf" && - trial.values[objectiveId] !== "-inf" - ) -} - const plotSlice = ( trials: Trial[], - objectiveId: number, - selected: string | null, - logXScale: boolean, + objectiveTarget: Target, + selected: SearchSpaceItem | null, logYScale: boolean, mode: string ) => { @@ -160,8 +147,8 @@ const plotSlice = ( b: 0, }, xaxis: { - title: selected || "", - type: logXScale ? "log" : "linear", + title: selected?.name || "", + type: selected !== null && isLogScale(selected) ? "log" : "linear", gridwidth: 1, automargin: true, }, @@ -174,34 +161,27 @@ const plotSlice = ( showlegend: false, template: mode === "dark" ? plotlyDarkTemplate : {}, } - - const filteredTrials = trials.filter((t) => - filterFunc(t, objectiveId, selected) - ) - - if (filteredTrials.length === 0 || selected === null) { + if (selected === null) { + plotly.react(plotDomId, [], layout) + return + } + if (trials.length === 0) { plotly.react(plotDomId, [], layout) return } - const objectiveValues: number[] = filteredTrials.map( - (t) => t.values![objectiveId] as number + const objectiveValues: number[] = trials.map( + (t) => objectiveTarget.getTargetValue(t) as number ) - const valueStrings = filteredTrials.map((t) => { - return t.params.find((p) => p.name == selected)!.value - }) + const paramTarget = new Target("params", selected.name) + const values = trials.map((t) => paramTarget.getTargetValue(t) as number) - const trialNumbers: number[] = filteredTrials.map((t) => t.number) - - const isnum = valueStrings.every((v) => { - return !isNaN(Number(v)) - }) - if (isnum) { - const valuesNum: number[] = valueStrings.map((v) => parseFloat(v)) + const trialNumbers: number[] = trials.map((t) => t.number) + if (selected.distribution.type !== "CategoricalDistribution") { const trace: plotly.Data[] = [ { type: "scatter", - x: valuesNum, + x: values, y: objectiveValues, mode: "markers", marker: { @@ -219,23 +199,19 @@ const plotSlice = ( }, ] layout["xaxis"] = { - title: selected, - type: logXScale ? "log" : "linear", + title: selected.name, + type: selected.distribution.log ? "log" : "linear", gridwidth: 1, automargin: true, // Otherwise the label is outside of the plot } plotly.react(plotDomId, trace, layout) } else { - const vocabSet = new Set(valueStrings) - const vocabArr = Array.from(vocabSet) - const valuesCategorical: number[] = valueStrings.map((v) => - vocabArr.findIndex((vocab) => v === vocab) - ) + const vocabArr = selected.distribution.choices.map((c) => c.value) const tickvals: number[] = vocabArr.map((v, i) => i) const trace: plotly.Data[] = [ { type: "scatter", - x: valuesCategorical, + x: values, y: objectiveValues, mode: "markers", marker: { @@ -253,8 +229,8 @@ const plotSlice = ( }, ] layout["xaxis"] = { - title: selected, - type: logXScale ? "log" : "linear", + title: selected.name, + type: "linear", gridwidth: 1, tickvals: tickvals, ticktext: vocabArr, diff --git a/optuna_dashboard/ts/components/PreferenceDialog.tsx b/optuna_dashboard/ts/components/PreferenceDialog.tsx index 79261f7f..399144e8 100644 --- a/optuna_dashboard/ts/components/PreferenceDialog.tsx +++ b/optuna_dashboard/ts/components/PreferenceDialog.tsx @@ -49,6 +49,15 @@ export const usePreferenceDialog = ( [event.target.name]: event.target.checked, }) } + const renderSelectBox = (onChange: (e) => void): ReactNode => ( + + ) const renderPreferenceDialog = () => { return ( diff --git a/optuna_dashboard/ts/components/StudyDetailBeta.tsx b/optuna_dashboard/ts/components/StudyDetailBeta.tsx index 2152cf90..a98c3d04 100644 --- a/optuna_dashboard/ts/components/StudyDetailBeta.tsx +++ b/optuna_dashboard/ts/components/StudyDetailBeta.tsx @@ -24,7 +24,7 @@ import { useStudySummaryValue, } from "../state" import { TrialTable } from "./TrialTable" -import { AppDrawer } from "./AppDrawer" +import { AppDrawer, PageId } from "./AppDrawer" import { GraphParallelCoordinate } from "./GraphParallelCoordinate" import { Contour } from "./GraphContour" import { GraphHyperparameterImportanceBeta } from "./GraphHyperparameterImportances" diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 44b180c7..8e20df96 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -162,7 +162,10 @@ const TrialListDetail: FC<{ Params = [ - {trial.params.map((p) => `${p.name}: ${p.value}`).join(", ")}] + {trial.params + .map((p) => `${p.name}: ${p.param_external_value}`) + .join(", ")} + ] Started At ={" "} diff --git a/optuna_dashboard/ts/components/TrialTable.tsx b/optuna_dashboard/ts/components/TrialTable.tsx index ba271bf1..e88aebe6 100644 --- a/optuna_dashboard/ts/components/TrialTable.tsx +++ b/optuna_dashboard/ts/components/TrialTable.tsx @@ -139,25 +139,28 @@ export const TrialTable: FC<{ studyDetail?.intersection_search_space.length ) { studyDetail?.intersection_search_space.forEach((s) => { - const sortable = s.distribution !== "CategoricalDistribution" - const filterable = s.distribution === "CategoricalDistribution" + const sortable = s.distribution.type !== "CategoricalDistribution" + const filterable = s.distribution.type === "CategoricalDistribution" columns.push({ field: "params", label: `Param ${s.name}`, toCellValue: (i) => - trials[i].params.find((p) => p.name === s.name)?.value || null, + trials[i].params.find((p) => p.name === s.name) + ?.param_external_value || null, sortable: sortable, filterable: filterable, less: (firstEl, secondEl): number => { - const firstVal = firstEl.params.find((p) => p.name === s.name)?.value + const firstVal = firstEl.params.find( + (p) => p.name === s.name + )?.param_internal_value const secondVal = secondEl.params.find( (p) => p.name === s.name - )?.value + )?.param_internal_value if (firstVal === secondVal) { return 0 } else if (firstVal && secondVal) { - return Number(firstVal) < Number(secondVal) ? 1 : -1 + return firstVal < secondVal ? 1 : -1 } else if (firstVal) { return -1 } else { @@ -171,7 +174,9 @@ export const TrialTable: FC<{ field: "params", label: "Params", toCellValue: (i) => - trials[i].params.map((p) => p.name + ": " + p.value).join(", "), + trials[i].params + .map((p) => p.name + ": " + p.param_external_value) + .join(", "), }) } diff --git a/optuna_dashboard/ts/searchSpace.ts b/optuna_dashboard/ts/searchSpace.ts new file mode 100644 index 00000000..729f1f51 --- /dev/null +++ b/optuna_dashboard/ts/searchSpace.ts @@ -0,0 +1,37 @@ +import { useMemo } from "react" + +export const mergeUnionSearchSpace = ( + unionSearchSpace: SearchSpaceItem[] +): SearchSpaceItem[] => { + const knownElements = new Map() + unionSearchSpace.forEach((s) => { + const d = knownElements.get(s.name) + if (d === undefined) { + knownElements.set(s.name, s.distribution) + return + } + if ( + d.type === "CategoricalDistribution" || + s.distribution.type === "CategoricalDistribution" + ) { + // CategoricalDistribution.choices will never be changed + return + } + d.low = Math.min(d.low, s.distribution.low) + d.high = Math.max(d.low, s.distribution.high) + knownElements.set(s.name, d) + }) + return Array.from(knownElements.keys()) + .sort((a, b) => (a > b ? 1 : a < b ? -1 : 0)) + .map((name) => ({ + name: name, + distribution: knownElements.get(name) as Distribution, + })) +} + +export const useMergedUnionSearchSpace = ( + unionSearchSpaces?: SearchSpaceItem[] +): SearchSpaceItem[] => + useMemo(() => { + return mergeUnionSearchSpace(unionSearchSpaces || []) + }, [unionSearchSpaces]) diff --git a/optuna_dashboard/ts/trialFilter.ts b/optuna_dashboard/ts/trialFilter.ts new file mode 100644 index 00000000..204fa817 --- /dev/null +++ b/optuna_dashboard/ts/trialFilter.ts @@ -0,0 +1,152 @@ +import { useMemo } from "react" +import { mergeUnionSearchSpace } from "./searchSpace" + +type TargetKind = "objective" | "user_attr" | "params" + +export class Target { + kind: TargetKind + key: number | string + + constructor(kind: TargetKind, key: number | string) { + this.kind = kind + this.key = key + } + + validate(): boolean { + if (this.kind === "objective") { + if (typeof this.key !== "number") { + return false + } + } else if (this.kind === "user_attr") { + if (typeof this.key !== "string") { + return false + } + } else if (this.kind === "params") { + if (typeof this.key !== "string") { + return false + } + } + return true + } + + toLabel(objectiveNames?: string[]): string { + if (this.kind === "objective") { + const objectiveId: number = this.key as number + if (objectiveNames !== undefined && objectiveNames.length > objectiveId) { + return objectiveNames[objectiveId] + } + return `Objective ${objectiveId}` + } else if (this.kind === "user_attr") { + return `User Attribute ${this.key}` + } else { + return `Param ${this.key}` + } + } + + getObjectiveId(): number | null { + if (this.kind !== "objective") { + return null + } + return this.key as number + } + + getTargetValue(trial: Trial): number | null { + if (!this.validate()) { + return null + } + if (this.kind === "objective") { + const objectiveId = this.getObjectiveId() + if ( + objectiveId === null || + trial.values === undefined || + trial.values.length <= objectiveId + ) { + return null + } + const value = trial.values[objectiveId] + if (value === "inf" || value === "-inf") { + return null + } + return value + } else if (this.kind === "user_attr") { + const attr = trial.user_attrs.find((attr) => attr.key === this.key) + if (attr === undefined) { + return null + } + const value = Number(attr.value) + if (value === undefined) { + return null + } + return value + } else if (this.kind === "params") { + const param = trial.params.find((p) => p.name === this.key) + if (param === undefined) { + return null + } + return param.param_internal_value + } + return null + } +} + +export const useFilteredTrials = ( + study: StudyDetail | null, + targets: Target[], + filterComplete: boolean, + filterPruned: boolean +): Trial[] => + useMemo(() => { + if (study === null) { + return [] + } + return study.trials.filter((t) => { + if (t.state !== "Complete" && t.state !== "Pruned") { + return false + } + if (t.state === "Complete" && filterComplete) { + return false + } + if (t.state === "Pruned" && filterPruned) { + return false + } + return targets.every((target) => target.getTargetValue(t) !== null) + }) + }, [study?.trials, targets, filterComplete, filterPruned]) + +export const useObjectiveTargets = (study: StudyDetail | null): Target[] => + useMemo(() => { + if (study !== null) { + return study.directions.map((v, i) => new Target("objective", i)) + } else { + return [new Target("objective", 0)] + } + }, [study?.directions]) + +export const useParamTargets = ( + study: StudyDetail | null +): [Target[], SearchSpaceItem[]] => + useMemo<[Target[], SearchSpaceItem[]]>(() => { + if (study !== null) { + const searchSpace = mergeUnionSearchSpace(study.union_search_space) + const targets = searchSpace.map((s) => new Target("params", s.name)) + return [targets, searchSpace] + } else { + return [[], []] + } + }, [study?.union_search_space]) + +export const useObjectiveAndSystemAttrTargets = ( + study: StudyDetail | null +): Target[] => + useMemo(() => { + if (study !== null) { + return [ + ...study.directions.map((v, i) => new Target("objective", i)), + ...study.union_user_attrs + .filter((attr) => attr.sortable) + .map((attr) => new Target("user_attr", attr.key)), + ] + } else { + return [new Target("objective", 0)] + } + }, [study?.directions, study?.union_user_attrs]) diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 98c42ed7..440789ba 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -11,17 +11,32 @@ type TrialValueNumber = number | "inf" | "-inf" type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type StudyDirection = "maximize" | "minimize" | "not_set" -type Distribution = - | "FloatDistribution" - | "IntDistribution" - | "UniformDistribution" - | "LogUniformDistribution" - | "DiscreteUniformDistribution" - | "IntUniformDistribution" - | "IntLogUniformDistribution" - | "CategoricalDistribution" -type PageId = "history" | "analytics" | "trialTable" | "trialList" | "note" +type FloatDistribution = { + type: "FloatDistribution" + low: number + high: number + step: number + log: boolean +} + +type IntDistribution = { + type: "IntDistribution" + low: number + high: number + step: number + log: boolean +} + +type CategoricalDistribution = { + type: "CategoricalDistribution" + choices: { pytype: string; value: string }[] +} + +type Distribution = + | FloatDistribution + | IntDistribution + | CategoricalDistribution type GraphVisibility = { history: boolean @@ -41,7 +56,10 @@ type TrialIntermediateValue = { type TrialParam = { name: string - value: string + param_internal_value: number + param_external_value: string + param_external_type: string + distribution: Distribution } type ParamImportance = { @@ -50,7 +68,7 @@ type ParamImportance = { distribution: Distribution } -type SearchSpace = { +type SearchSpaceItem = { name: string distribution: Distribution } @@ -101,8 +119,8 @@ type StudyDetail = { datetime_start: Date best_trials: Trial[] trials: Trial[] - intersection_search_space: SearchSpace[] - union_search_space: SearchSpace[] + intersection_search_space: SearchSpaceItem[] + union_search_space: SearchSpaceItem[] union_user_attrs: AttributeSpec[] has_intermediate_values: boolean note: Note diff --git a/typescript_tests/TrialTable.test.tsx b/typescript_tests/TrialTable.test.tsx index c443df13..b9cd6035 100644 --- a/typescript_tests/TrialTable.test.tsx +++ b/typescript_tests/TrialTable.test.tsx @@ -6,7 +6,14 @@ import { TrialTable } from "../optuna_dashboard/ts/components/TrialTable" afterEach(cleanup) -const trials = [ +const dummyDistribution: FloatDistribution = { + type: "FloatDistribution", + low: 0, + high: 10, + step: 1, + log: false, +} +const trials: Trial[] = [ { trial_id: 1, study_id: 0, @@ -17,8 +24,20 @@ const trials = [ datetime_start: new Date("2021-06-15T00:00:00"), datetime_complete: new Date("2021-06-15T00:00:01"), params: [ - { name: "x", value: "1" }, - { name: "y", value: "2" }, + { + name: "x", + param_internal_value: 1, + param_external_value: "1", + param_external_type: "float", + distribution: dummyDistribution, + }, + { + name: "y", + param_internal_value: 2, + param_external_value: "2", + param_external_type: "float", + distribution: dummyDistribution, + }, ], user_attrs: [], system_attrs: [], @@ -37,8 +56,20 @@ const trials = [ datetime_start: new Date("2021-06-15T00:00:01"), datetime_complete: new Date("2021-06-15T00:00:03"), params: [ - { name: "x", value: "2" }, - { name: "y", value: "1" }, + { + name: "x", + param_internal_value: 1, + param_external_value: "1", + param_external_type: "float", + distribution: dummyDistribution, + }, + { + name: "y", + param_internal_value: 2, + param_external_value: "2", + param_external_type: "float", + distribution: dummyDistribution, + }, ], user_attrs: [], system_attrs: [], @@ -61,21 +92,21 @@ const studyDetail: StudyDetail = { intersection_search_space: [ { name: "x", - distribution: "FloatDistribution" as Distribution, + distribution: dummyDistribution, }, { name: "y", - distribution: "FloatDistribution" as Distribution, + distribution: dummyDistribution, }, ], union_search_space: [ { name: "x", - distribution: "FloatDistribution" as Distribution, + distribution: dummyDistribution, }, { name: "y", - distribution: "FloatDistribution" as Distribution, + distribution: dummyDistribution, }, ], union_user_attrs: [ diff --git a/typescript_tests/searchSpace.test.ts b/typescript_tests/searchSpace.test.ts new file mode 100644 index 00000000..fa36be7c --- /dev/null +++ b/typescript_tests/searchSpace.test.ts @@ -0,0 +1,34 @@ +import { mergeUnionSearchSpace } from "../optuna_dashboard/ts/searchSpace" + +global.URL.createObjectURL = jest.fn() + +it("Aggregate SearchSpaceItem", () => { + const aggregated = mergeUnionSearchSpace([ + { + name: "float1", + distribution: { + type: "FloatDistribution", + low: 0, + high: 5, + step: 1, + log: true, + }, + }, + { + name: "float1", + distribution: { + type: "FloatDistribution", + low: 5, + high: 10, + step: 1, + log: true, + }, + }, + ]) + + expect(aggregated.length).toBe(1) + // @ts-ignore + expect(aggregated[0].distribution.low as number).toBe(0) + // @ts-ignore + expect(aggregated[0].distribution.high as number).toBe(10) +})