From a853ccd01f37a0b8c445aa3eb4a43680d2f97aca Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 12 Mar 2022 01:31:29 +0900 Subject: [PATCH] Fix bug when given inf objective value --- optuna_dashboard/_serializer.py | 11 +++-- optuna_dashboard/static/apiClient.ts | 2 +- .../static/components/GraphEdf.tsx | 12 ++++-- .../static/components/GraphHistory.tsx | 42 ++++++++++++------- .../components/GraphIntermediateValues.tsx | 5 ++- .../components/GraphParallelCoordinate.tsx | 20 ++++++--- .../static/components/GraphParetoFront.tsx | 26 +++++++----- .../static/components/GraphSlice.tsx | 28 ++++++++++--- optuna_dashboard/static/types/index.d.ts | 4 +- visual_regression_test.py | 13 ++++++ 10 files changed, 113 insertions(+), 50 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 1252ce4c..8b25e655 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -1,8 +1,10 @@ import json +import math from typing import Any from typing import Dict from typing import List from typing import Tuple +from typing import Union from optuna.distributions import BaseDistribution from optuna.study import StudySummary @@ -27,7 +29,7 @@ IntermediateValue = TypedDict( "IntermediateValue", { "step": int, - "value": float, + "value": Union[float, str], }, ) TrialParam = TypedDict( @@ -53,7 +55,10 @@ def serialize_attrs(attrs: Dict[str, Any]) -> List[Attribute]: def serialize_intermediate_values(values: Dict[int, float]) -> List[IntermediateValue]: - return [{"step": step, "value": value} for step, value in values.items()] + return [ + {"step": step, "value": "inf" if math.isinf(value) else value} + for step, value in values.items() + ] def serialize_trial_params(params: Dict[str, Any]) -> List[TrialParam]: @@ -120,7 +125,7 @@ def serialize_frozen_trial(study_id: int, trial: FrozenTrial) -> Dict[str, Any]: } if trial.values is not None: - serialized["values"] = trial.values + serialized["values"] = ["inf" if math.isinf(v) else v for v in trial.values] if trial.datetime_start is not None: serialized["datetime_start"] = trial.datetime_start.isoformat() diff --git a/optuna_dashboard/static/apiClient.ts b/optuna_dashboard/static/apiClient.ts index add7a531..e5dba916 100644 --- a/optuna_dashboard/static/apiClient.ts +++ b/optuna_dashboard/static/apiClient.ts @@ -7,7 +7,7 @@ interface TrialResponse { study_id: number number: number state: TrialState - values?: number[] + values?: (number | "inf")[] intermediate_values: TrialIntermediateValue[] datetime_start?: string datetime_complete?: string diff --git a/optuna_dashboard/static/components/GraphEdf.tsx b/optuna_dashboard/static/components/GraphEdf.tsx index f799e04a..cd6e74d4 100644 --- a/optuna_dashboard/static/components/GraphEdf.tsx +++ b/optuna_dashboard/static/components/GraphEdf.tsx @@ -64,15 +64,19 @@ export const Edf: FC<{ ) } +const filterFunc = (trial: Trial, objectiveId: number): boolean => { + return trial.state !== "Complete" || trial.values![objectiveId] !== "inf" +} + const plotEdf = (study: StudyDetail, objectiveId: number, mode: string) => { if (document.getElementById(plotDomId) === null) { return } const trials: Trial[] = study ? study.trials : [] - const completedTrials = trials.filter((t) => t.state === "Complete") + const filteredTrials = trials.filter((t) => filterFunc(t, objectiveId)) - if (completedTrials.length === 0) { + if (filteredTrials.length === 0) { plotly.react(plotDomId, []) return } @@ -80,7 +84,7 @@ const plotEdf = (study: StudyDetail, objectiveId: number, mode: string) => { const target_name = "Objective Value" const target = (t: Trial): number => { - return t.values![objectiveId] + return t.values![objectiveId] as number } const layout: Partial = { @@ -99,7 +103,7 @@ const plotEdf = (study: StudyDetail, objectiveId: number, mode: string) => { template: mode === "dark" ? plotlyDarkTemplate : {}, } - const values = completedTrials.map((t) => target(t)) + const values = filteredTrials.map((t) => target(t)) const numValues = values.length const minX = Math.min(...values) const maxX = Math.max(...values) diff --git a/optuna_dashboard/static/components/GraphHistory.tsx b/optuna_dashboard/static/components/GraphHistory.tsx index 9607c63f..c7b15bc5 100644 --- a/optuna_dashboard/static/components/GraphHistory.tsx +++ b/optuna_dashboard/static/components/GraphHistory.tsx @@ -168,6 +168,18 @@ export const GraphHistory: FC<{ ) } +const filterFunc = (trial: Trial, objectiveId: number): boolean => { + if (trial.state !== "Complete" && trial.state !== "Pruned") { + return false + } + if (trial.values === undefined) { + return false + } + return ( + trial.values.length > objectiveId && trial.values[objectiveId] !== "inf" + ) +} + const plotHistory = ( study: StudyDetail, objectiveId: number, @@ -198,11 +210,7 @@ const plotHistory = ( template: mode === "dark" ? plotlyDarkTemplate : {}, } - let filteredTrials = study.trials.filter( - (t) => - t.state === "Complete" || - (t.state === "Pruned" && t.values && t.values.length > 0) - ) + let filteredTrials = study.trials.filter((t) => filterFunc(t, objectiveId)) if (filterCompleteTrial) { filteredTrials = filteredTrials.filter((t) => t.state !== "Complete") } @@ -215,22 +223,22 @@ const plotHistory = ( } const trialsForLinePlot: Trial[] = [] let currentBest: number | null = null - filteredTrials.forEach((item) => { + filteredTrials.forEach((t) => { if (currentBest === null) { - currentBest = item.values![objectiveId] - trialsForLinePlot.push(item) + currentBest = t.values![objectiveId] as number + trialsForLinePlot.push(t) } else if ( study.directions[objectiveId] === "maximize" && - item.values![objectiveId] > currentBest + t.values![objectiveId] > currentBest ) { - currentBest = item.values![objectiveId] - trialsForLinePlot.push(item) + currentBest = t.values![objectiveId] as number + trialsForLinePlot.push(t) } else if ( study.directions[objectiveId] === "minimize" && - item.values![objectiveId] < currentBest + t.values![objectiveId] < currentBest ) { - currentBest = item.values![objectiveId] - trialsForLinePlot.push(item) + currentBest = t.values![objectiveId] as number + trialsForLinePlot.push(t) } }) @@ -245,14 +253,16 @@ const plotHistory = ( const xForLinePlot = trialsForLinePlot.map(getAxisX) xForLinePlot.push(getAxisX(filteredTrials[filteredTrials.length - 1])) const yForLinePlot = trialsForLinePlot.map( - (t: Trial): number => t.values![objectiveId] + (t: Trial): number => t.values![objectiveId] as number ) yForLinePlot.push(yForLinePlot[yForLinePlot.length - 1]) const plotData: Partial[] = [ { x: filteredTrials.map(getAxisX), - y: filteredTrials.map((t: Trial): number => t.values![objectiveId]), + y: filteredTrials.map( + (t: Trial): number => t.values![objectiveId] as number + ), mode: "markers", type: "scatter", }, diff --git a/optuna_dashboard/static/components/GraphIntermediateValues.tsx b/optuna_dashboard/static/components/GraphIntermediateValues.tsx index 09f4c146..2a8cbf5e 100644 --- a/optuna_dashboard/static/components/GraphIntermediateValues.tsx +++ b/optuna_dashboard/static/components/GraphIntermediateValues.tsx @@ -54,9 +54,10 @@ const plotIntermediateValue = (trials: Trial[], mode: string) => { (t.state === "Pruned" && t.values && t.values.length > 0) ) const plotData: Partial[] = filteredTrials.map((trial) => { + const values = trial.intermediate_values.filter((iv) => iv.value !== "inf") return { - x: trial.intermediate_values.map((iv) => iv.step), - y: trial.intermediate_values.map((iv) => iv.value), + x: values.map((iv) => iv.step), + y: values.map((iv) => iv.value), mode: "lines+markers", type: "scatter", name: `trial #${trial.number}`, diff --git a/optuna_dashboard/static/components/GraphParallelCoordinate.tsx b/optuna_dashboard/static/components/GraphParallelCoordinate.tsx index 2b1f6ef4..8c008203 100644 --- a/optuna_dashboard/static/components/GraphParallelCoordinate.tsx +++ b/optuna_dashboard/static/components/GraphParallelCoordinate.tsx @@ -65,6 +65,18 @@ export const GraphParallelCoordinate: FC<{ ) } +const filterFunc = (trial: Trial, objectiveId: number): boolean => { + if (trial.state !== "Complete" && trial.state !== "Pruned") { + return false + } + if (trial.values === undefined) { + return false + } + return ( + trial.values.length > objectiveId && trial.values[objectiveId] !== "inf" + ) +} + const plotCoordinate = ( study: StudyDetail, objectiveId: number, @@ -89,11 +101,7 @@ const plotCoordinate = ( return } - const filteredTrials = study.trials.filter( - (t) => - t.state === "Complete" || - (t.state === "Pruned" && t.values && t.values.length > 0) - ) + const filteredTrials = study.trials.filter((t) => filterFunc(t, objectiveId)) const maxLabelLength = 40 const breakLength = maxLabelLength / 2 @@ -115,7 +123,7 @@ const plotCoordinate = ( // Intersection param names const objectiveValues: number[] = filteredTrials.map( - (t) => t.values![objectiveId] + (t) => t.values![objectiveId] as number ) const dimensions = [ { diff --git a/optuna_dashboard/static/components/GraphParetoFront.tsx b/optuna_dashboard/static/components/GraphParetoFront.tsx index eeae1c6c..080de53a 100644 --- a/optuna_dashboard/static/components/GraphParetoFront.tsx +++ b/optuna_dashboard/static/components/GraphParetoFront.tsx @@ -85,6 +85,10 @@ export const GraphParetoFront: FC<{ ) } +const filterFunc = (trial: Trial): boolean => { + return trial.state !== "Complete" || trial.values!.every((v) => v !== "inf") +} + const plotParetoFront = ( study: StudyDetail, objectiveXId: number, @@ -106,18 +110,20 @@ const plotParetoFront = ( } const trials: Trial[] = study ? study.trials : [] - const completedTrials = trials.filter((t) => t.state === "Complete") + const filteredTrials = trials.filter(filterFunc) - if (completedTrials.length === 0) { + if (filteredTrials.length === 0) { plotly.react(plotDomId, [], layout) return } const normalizedValues: number[][] = [] - completedTrials.forEach((t) => { + filteredTrials.forEach((t) => { if (t.values && t.values.length === study.directions.length) { - const trialValues = t.values.map((v: number, i: number) => { - return study.directions[i] === "minimize" ? v : -v + const trialValues = t.values.map((v, i) => { + return study.directions[i] === "minimize" + ? (v as number) + : (-v as number) }) normalizedValues.push(trialValues) } @@ -144,11 +150,11 @@ const plotParetoFront = ( const plotData: Partial[] = [ { type: "scatter", - x: completedTrials.map((t: Trial): number => { - return t.values![objectiveXId] + x: filteredTrials.map((t: Trial): number => { + return t.values![objectiveXId] as number }), - y: completedTrials.map((t: Trial): number => { - return t.values![objectiveYId] + y: filteredTrials.map((t: Trial): number => { + return t.values![objectiveYId] as number }), mode: "markers", xaxis: "Objective X", @@ -156,7 +162,7 @@ const plotParetoFront = ( marker: { color: pointColors, }, - text: completedTrials.map( + text: filteredTrials.map( (t: Trial): string => `Trial (number=${t.number})` ), hovertemplate: "%{text}", diff --git a/optuna_dashboard/static/components/GraphSlice.tsx b/optuna_dashboard/static/components/GraphSlice.tsx index b4a10198..06591132 100644 --- a/optuna_dashboard/static/components/GraphSlice.tsx +++ b/optuna_dashboard/static/components/GraphSlice.tsx @@ -129,6 +129,25 @@ export const GraphSlice: FC<{ ) } +const filterFunc = ( + trial: Trial, + objectiveId: number, + selected: string | null +): boolean => { + if (trial.state !== "Complete" && trial.state !== "Pruned") { + return false + } + if (trial.params.find((p) => p.name == selected) === undefined) { + return false + } + if (trial.values === undefined) { + return false + } + return ( + trial.values.length > objectiveId && trial.values[objectiveId] !== "inf" + ) +} + const plotSlice = ( trials: Trial[], objectiveId: number, @@ -164,11 +183,8 @@ const plotSlice = ( template: mode === "dark" ? plotlyDarkTemplate : {}, } - const filteredTrials = trials.filter( - (t) => - (t.state === "Complete" || - (t.state === "Pruned" && t.values && t.values.length > 0)) && - t.params.find((p) => p.name == selected) !== undefined + const filteredTrials = trials.filter((t) => + filterFunc(t, objectiveId, selected) ) if (filteredTrials.length === 0 || selected === null) { @@ -177,7 +193,7 @@ const plotSlice = ( } const objectiveValues: number[] = filteredTrials.map( - (t) => t.values![objectiveId] + (t) => t.values![objectiveId] as number ) const valueStrings = filteredTrials.map((t) => { return t.params.find((p) => p.name == selected)!.value diff --git a/optuna_dashboard/static/types/index.d.ts b/optuna_dashboard/static/types/index.d.ts index 614b78f0..80268d93 100644 --- a/optuna_dashboard/static/types/index.d.ts +++ b/optuna_dashboard/static/types/index.d.ts @@ -21,7 +21,7 @@ type Distribution = declare interface TrialIntermediateValue { step: number - value: number + value: number | "inf" } declare interface TrialParam { @@ -50,7 +50,7 @@ declare interface Trial { study_id: number number: number state: TrialState - values?: number[] + values?: (number | "inf")[] intermediate_values: TrialIntermediateValue[] datetime_start?: Date datetime_complete?: Date diff --git a/visual_regression_test.py b/visual_regression_test.py index 2da2ab31..ad3ff246 100644 --- a/visual_regression_test.py +++ b/visual_regression_test.py @@ -1,5 +1,6 @@ import argparse import asyncio +import math import os import sys import threading @@ -84,6 +85,18 @@ def create_dummy_storage() -> optuna.storages.InMemoryStorage: study.optimize(objective_single_dynamic, n_trials=50) + # Single objective study with 'inf' value + study = optuna.create_study(study_name="single-inf", storage=storage) + + def objective_single_inf(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if x > 0: + return math.inf + else: + return x ** 2 + + study.optimize(objective_single_inf, n_trials=50) + # Multi-objective study study = optuna.create_study( study_name="multi-objective",