From ea3a986915f743b49c9ee887bf4ca2a1691816a8 Mon Sep 17 00:00:00 2001 From: HideakiImamura Date: Thu, 12 Oct 2023 15:46:53 +0900 Subject: [PATCH 01/26] Add rank plot --- optuna_dashboard/ts/components/GraphRank.tsx | 343 ++++++++++++++++++ .../ts/components/StudyDetail.tsx | 6 + 2 files changed, 349 insertions(+) create mode 100644 optuna_dashboard/ts/components/GraphRank.tsx diff --git a/optuna_dashboard/ts/components/GraphRank.tsx b/optuna_dashboard/ts/components/GraphRank.tsx new file mode 100644 index 00000000..17731101 --- /dev/null +++ b/optuna_dashboard/ts/components/GraphRank.tsx @@ -0,0 +1,343 @@ +import * as plotly from "plotly.js-dist-min" +import React, { FC, useEffect, useState } from "react" +import { + Grid, + FormControl, + FormLabel, + MenuItem, + Select, + Typography, + SelectChangeEvent, + useTheme, + Box, +} from "@mui/material" +import { plotlyDarkTemplate } from "./PlotlyDarkMode" +import { makeHovertext } from "../graphUtil" +import { useMergedUnionSearchSpace } from "../searchSpace" + +const PADDING_RATIO = 0.05 +const plotDomId = "graph-rank" + +interface AxisInfo { + name: string + range: [number, number] + is_log: boolean + is_cat: boolean +} + +interface RankPlotInfo { + xaxis: AxisInfo + yaxis: AxisInfo + xvalues: (string | number)[] + yvalues: (string | number)[] + zvalues: number[] + colors: number[] + hovertext: string[] +} + +export const GraphRank: FC<{ + study: StudyDetail | null +}> = ({ study = null }) => { + const theme = useTheme() + const [objectiveId, setobjectiveId] = useState(0) + const searchSpace = useMergedUnionSearchSpace(study?.union_search_space) + const [xParam, setXParam] = useState(null) + const [yParam, setYParam] = useState(null) + const objectiveNames: string[] = study?.objective_names || [] + + if (xParam == null && searchSpace.length > 0) { + setXParam(searchSpace[0]) + } + if (yParam == null && searchSpace.length > 1) { + setYParam(searchSpace[1]) + } + + const handleObjectiveChange = (event: SelectChangeEvent) => { + setobjectiveId(Number(event.target.value)) + } + const handleXParamChange = (event: SelectChangeEvent) => { + const param = searchSpace.find((item) => item.name === event.target.value) + setXParam(param || null) + } + const handleYParamChange = (event: SelectChangeEvent) => { + const param = searchSpace.find((item) => item.name === event.target.value) + setYParam(param || null) + } + + const rankPlotInfo = getRankPlotInfo(study, objectiveId, xParam, yParam) + + useEffect(() => { + if (study != null) { + plotRank(rankPlotInfo, theme.palette.mode) + } + }, [study, theme.palette.mode]) + + const space: SearchSpaceItem[] = study ? study.union_search_space : [] + + return ( + + + + Rank + + {study !== null && study.directions.length !== 1 ? ( + + Objective: + + + ) : null} + {study !== null && space.length > 0 ? ( + + + x: + + + + y: + + + + ) : null} + + + + + + ) +} + +const getRankPlotInfo = ( + study: StudyDetail | null, + objectiveId: number, + xParam: SearchSpaceItem | null, + yParam: SearchSpaceItem | null +): RankPlotInfo | null => { + if (study === null) { + return null + } + + const trials = study.trials + const filtered_trials = trials.filter(filterFunc) + if (filtered_trials.length < 2 || xParam == null || yParam == null) { + return null + } + + const xAxis = getAxisInfo(filtered_trials, xParam) + const yAxis = getAxisInfo(filtered_trials, yParam) + + const xValues: number[] = [] + const yValues: number[] = [] + const zValues: number[] = [] + const hovertext: string[] = [] + filtered_trials.forEach((trial) => { + const xValue = + trial.params.find((p) => p.name === xAxis.name)?.param_internal_value || + null + const yValue = + trial.params.find((p) => p.name === yAxis.name)?.param_internal_value || + null + if (trial.values === undefined || xValue === null || yValue === null) { + return + } + const zValue = Number(trial.values[objectiveId]) + xValues.push(xValue) + yValues.push(yValue) + zValues.push(zValue) + hovertext.push(makeHovertext(trial)) + }) + + const colors = getColors(zValues) + + return { + xaxis: xAxis, + yaxis: yAxis, + xvalues: xValues, + yvalues: yValues, + zvalues: zValues, + colors, + hovertext, + } +} + +const filterFunc = (trial: Trial): boolean => { + return trial.state === "Complete" && trial.values !== undefined +} + +const getAxisInfo = (trials: Trial[], param: SearchSpaceItem): AxisInfo => { + if (param.distribution.type === "CategoricalDistribution") { + return getAxisInfoForCategorical(trials, param.name, param.distribution) + } else { + return getAxisInfoForNumerical(trials, param.name, param.distribution) + } +} + +const getAxisInfoForCategorical = ( + trials: Trial[], + param: string, + distribution: CategoricalDistribution +): AxisInfo => { + const values = trials.map( + (trial) => + trial.params.find((p) => p.name === param)?.param_internal_value || null + ) + const isDynamic = values.some((v) => v === null) + const span = distribution.choices.length - (isDynamic ? 2 : 1) + const padding = span * PADDING_RATIO + const min = -padding + const max = span + padding + + return { + name: param, + range: [min, max], + is_log: false, + is_cat: true, + } +} + +const getAxisInfoForNumerical = ( + trials: Trial[], + param: string, + distribution: FloatDistribution | IntDistribution +): AxisInfo => { + const values = trials.map( + (trial) => + trial.params.find((p) => p.name === param)?.param_internal_value || null + ) + const non_null_values: number[] = [] + values.forEach((value) => { + if (value !== null) { + non_null_values.push(value) + } + }) + let min = Math.min(...non_null_values) + let max = Math.max(...non_null_values) + if (distribution.log) { + const padding = (Math.log10(max) - Math.log10(min)) * PADDING_RATIO + min = Math.pow(10, Math.log10(min) - padding) + max = Math.pow(10, Math.log10(max) + padding) + } else { + const padding = (max - min) * PADDING_RATIO + min = min - padding + max = max + padding + } + + return { + name: param, + range: [min, max], + is_log: distribution.log, + is_cat: false, + } +} + +const getColors = (values: number[]): number[] => { + const raw_ranks = getOrderWithSameOrderAveraging(values) + let color_idxs: number[] = [] + if (values.length > 2) { + color_idxs = raw_ranks.map((rank) => rank / (values.length - 1)) + } else { + color_idxs = [0.5] + } + return color_idxs +} + +const getOrderWithSameOrderAveraging = (values: number[]): number[] => { + const sorted_values = values.slice().sort() + const ranks: number[] = [] + values.forEach((value) => { + const first_index = sorted_values.indexOf(value) + const last_index = sorted_values.lastIndexOf(value) + const sum_of_the_value = sorted_values + .slice(first_index, last_index + 1) + .reduce((a, b) => a + b, 0) + const rank = sum_of_the_value / (last_index - first_index + 1) + ranks.push(rank) + }) + return ranks +} + +const plotRank = (rank_plot_info: RankPlotInfo | null, mode: string) => { + if (document.getElementById(plotDomId) === null) { + return + } + + if (rank_plot_info === null) { + plotly.react(plotDomId, [], { + template: mode === "dark" ? plotlyDarkTemplate : {}, + }) + return + } + + const xAxis = rank_plot_info.xaxis + const yAxis = rank_plot_info.yaxis + const layout: Partial = { + xaxis: { + title: xAxis.name, + type: xAxis.is_cat ? "category" : xAxis.is_log ? "log" : "linear", + }, + yaxis: { + title: yAxis.name, + type: yAxis.is_cat ? "category" : yAxis.is_log ? "log" : "linear", + }, + margin: { + l: 50, + t: 0, + r: 50, + b: 50, + }, + uirevision: "true", + template: mode === "dark" ? plotlyDarkTemplate : {}, + } + const plotData: Partial[] = [ + { + type: "scatter", + x: rank_plot_info.xvalues, + y: rank_plot_info.yvalues, + marker: { + color: rank_plot_info.colors, + colorscale: "Portland", + colorbar: { + title: "Rank", + }, + size: 10, + line: { + color: "Grey", + width: 0.5, + }, + }, + mode: "markers", + showlegend: false, + hovertemplate: "%{hovertext}", + hovertext: rank_plot_info.hovertext, + }, + ] + plotly.react(plotDomId, plotData, layout) +} diff --git a/optuna_dashboard/ts/components/StudyDetail.tsx b/optuna_dashboard/ts/components/StudyDetail.tsx index 1a8ced33..663ff399 100644 --- a/optuna_dashboard/ts/components/StudyDetail.tsx +++ b/optuna_dashboard/ts/components/StudyDetail.tsx @@ -27,6 +27,7 @@ import { GraphParallelCoordinate } from "./GraphParallelCoordinate" import { Contour } from "./GraphContour" import { GraphSlice } from "./GraphSlice" import { GraphEdf } from "./GraphEdf" +import { GraphRank } from "./GraphRank" import { TrialList } from "./TrialList" import { StudyHistory } from "./StudyHistory" import { PreferentialTrials } from "./PreferentialTrials" @@ -121,6 +122,11 @@ export const StudyDetail: FC<{ + + + + + Empirical Distribution of the Objective Value From 7c92b4a731e61332d813b54bfe9d6808494642e4 Mon Sep 17 00:00:00 2001 From: Naoto Mizuno Date: Thu, 12 Oct 2023 17:11:08 +0900 Subject: [PATCH 02/26] Add test for _one_side_trunc_norm_sampling --- python_tests/preferential/samplers/test_gp.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 python_tests/preferential/samplers/test_gp.py diff --git a/python_tests/preferential/samplers/test_gp.py b/python_tests/preferential/samplers/test_gp.py new file mode 100644 index 00000000..a249f4b2 --- /dev/null +++ b/python_tests/preferential/samplers/test_gp.py @@ -0,0 +1,29 @@ +import sys +from unittest.mock import patch + +import numpy as np +import pytest +import torch + + +if sys.version_info >= (3, 8): + from optuna_dashboard.preferential.samplers.gp import _one_side_trunc_norm_sampling +else: + pytest.skip("BoTorch dropped Python3.7 support", allow_module_level=True) + + +def test_one_side_trunc_norm_sampling() -> None: + for lower in np.linspace(-10, 10, 100): + assert _one_side_trunc_norm_sampling(torch.tensor([lower], dtype=torch.float64)) >= lower + + with patch.object(torch, "rand", return_value=torch.tensor([0.4], dtype=torch.float64)): + sampled_value = _one_side_trunc_norm_sampling(torch.tensor([0.1], dtype=torch.float64)) + assert np.allclose(sampled_value.numpy(), 0.899967154837563) + + with patch.object(torch, "rand", return_value=torch.tensor([0.8], dtype=torch.float64)): + sampled_value = _one_side_trunc_norm_sampling(torch.tensor([-2.3], dtype=torch.float64)) + assert np.allclose(sampled_value.numpy(), -0.8113606739551955) + + with patch.object(torch, "rand", return_value=torch.tensor([0.1], dtype=torch.float64)): + sampled_value = _one_side_trunc_norm_sampling(torch.tensor([5], dtype=torch.float64)) + assert np.allclose(sampled_value.numpy(), 5.426934003050024) From 926ebd3362813d518342af6999ae82ce047d76a7 Mon Sep 17 00:00:00 2001 From: Naoto Mizuno Date: Thu, 12 Oct 2023 17:34:47 +0900 Subject: [PATCH 03/26] Improve accuracy of _one_side_trunc_norm_sampling --- optuna_dashboard/preferential/samplers/gp.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 2e782b68..457379dc 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -50,15 +50,15 @@ def _orthants_MVN_Gibbs_sampling(cov_inv: Tensor, cycles: int, initial_sample: T def _one_side_trunc_norm_sampling(lower: Tensor) -> Tensor: - if lower > 4.0: - r = torch.clamp_min(torch.rand(torch.Size(()), dtype=torch.float64), min=1e-300) - return (lower * lower - 2 * r.log()).sqrt() - else: - SQRT2 = math.sqrt(2) - r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) - while 1 - r == 1: - r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) - return torch.erfinv(1 - r) * SQRT2 + r = torch.rand(torch.Size(()), dtype=torch.float64) + ret = -torch.special.ndtri(torch.exp(torch.special.log_ndtr(-lower) + r.log())) + + # If sampled random number is very small, `ret` becomes inf. + while torch.isinf(ret): + r = torch.rand(torch.Size(()), dtype=torch.float64) + ret = -torch.special.ndtri(torch.exp(torch.special.log_ndtr(-lower) + r.log())) + + return ret _orthants_MVN_Gibbs_sampling_jit = torch.jit.script(_orthants_MVN_Gibbs_sampling) From 82cc858a7929f1fb16e2200a21590edcf488738e Mon Sep 17 00:00:00 2001 From: Naoto Mizuno Date: Fri, 13 Oct 2023 11:08:51 +0900 Subject: [PATCH 04/26] Fix test for Python 3.7 --- python_tests/preferential/samplers/test_gp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tests/preferential/samplers/test_gp.py b/python_tests/preferential/samplers/test_gp.py index a249f4b2..136959a7 100644 --- a/python_tests/preferential/samplers/test_gp.py +++ b/python_tests/preferential/samplers/test_gp.py @@ -3,11 +3,11 @@ from unittest.mock import patch import numpy as np import pytest -import torch if sys.version_info >= (3, 8): from optuna_dashboard.preferential.samplers.gp import _one_side_trunc_norm_sampling + import torch else: pytest.skip("BoTorch dropped Python3.7 support", allow_module_level=True) From 4df81f221d98dcc97a996b0adb71696268ea231d Mon Sep 17 00:00:00 2001 From: HideakiImamura Date: Sun, 15 Oct 2023 23:35:01 +0900 Subject: [PATCH 05/26] Fix rank calculation logic --- optuna_dashboard/ts/components/GraphRank.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphRank.tsx b/optuna_dashboard/ts/components/GraphRank.tsx index 17731101..6030a722 100644 --- a/optuna_dashboard/ts/components/GraphRank.tsx +++ b/optuna_dashboard/ts/components/GraphRank.tsx @@ -275,9 +275,7 @@ const getOrderWithSameOrderAveraging = (values: number[]): number[] => { values.forEach((value) => { const first_index = sorted_values.indexOf(value) const last_index = sorted_values.lastIndexOf(value) - const sum_of_the_value = sorted_values - .slice(first_index, last_index + 1) - .reduce((a, b) => a + b, 0) + const sum_of_the_value = (first_index + last_index) * (last_index - first_index + 1) / 2 const rank = sum_of_the_value / (last_index - first_index + 1) ranks.push(rank) }) From d39a2a2a63b74aa7939071bed0b0f5c2e783c641 Mon Sep 17 00:00:00 2001 From: HideakiImamura Date: Fri, 20 Oct 2023 10:56:34 +0900 Subject: [PATCH 06/26] Run linter --- optuna_dashboard/ts/components/GraphRank.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/optuna_dashboard/ts/components/GraphRank.tsx b/optuna_dashboard/ts/components/GraphRank.tsx index 6030a722..b6e2fd1f 100644 --- a/optuna_dashboard/ts/components/GraphRank.tsx +++ b/optuna_dashboard/ts/components/GraphRank.tsx @@ -275,7 +275,8 @@ const getOrderWithSameOrderAveraging = (values: number[]): number[] => { values.forEach((value) => { const first_index = sorted_values.indexOf(value) const last_index = sorted_values.lastIndexOf(value) - const sum_of_the_value = (first_index + last_index) * (last_index - first_index + 1) / 2 + const sum_of_the_value = + ((first_index + last_index) * (last_index - first_index + 1)) / 2 const rank = sum_of_the_value / (last_index - first_index + 1) ranks.push(rank) }) From df13d1b0802c47d8f2440508d042f16408c1626b Mon Sep 17 00:00:00 2001 From: Victoria A <52001888+adjeiv@users.noreply.github.com> Date: Sat, 21 Oct 2023 17:13:46 +0100 Subject: [PATCH 07/26] Use ascending param to decide how to order infinite values --- optuna_dashboard/ts/components/DataGrid.tsx | 5 +++-- optuna_dashboard/ts/components/TrialTable.tsx | 16 ++++++++-------- standalone_app/src/components/DataGrid.tsx | 5 +++-- standalone_app/src/components/TrialTable.tsx | 16 ++++++++-------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/optuna_dashboard/ts/components/DataGrid.tsx b/optuna_dashboard/ts/components/DataGrid.tsx index 2d6a5670..d19ec9d9 100644 --- a/optuna_dashboard/ts/components/DataGrid.tsx +++ b/optuna_dashboard/ts/components/DataGrid.tsx @@ -28,7 +28,7 @@ interface DataGridColumn { field: keyof T label: string sortable?: boolean - less?: (a: T, b: T) => number + less?: (a: T, b: T, ascending: boolean) => number filterable?: boolean toCellValue?: (rowIndex: number) => string | React.ReactNode padding?: "normal" | "checkbox" | "none" @@ -358,7 +358,8 @@ function stableSort( const stabilizedThis = array.map((el, index) => [el, index] as [T, number]) stabilizedThis.sort((a, b) => { if (less) { - const result = order == "asc" ? -less(a[0], b[0]) : less(a[0], b[0]) + const ascending = order == "asc" + const result = ascending ? -less(a[0], b[0], ascending) : less(a[0], b[0], ascending) if (result !== 0) return result } else { const result = comparator(a[0], b[0]) diff --git a/optuna_dashboard/ts/components/TrialTable.tsx b/optuna_dashboard/ts/components/TrialTable.tsx index 8fcce63c..6090ee85 100644 --- a/optuna_dashboard/ts/components/TrialTable.tsx +++ b/optuna_dashboard/ts/components/TrialTable.tsx @@ -28,7 +28,7 @@ export const TrialTable: FC<{ field: "values", label: "Value", sortable: true, - less: (firstEl, secondEl): number => { + less: (firstEl, secondEl, ascending): number => { const firstVal = firstEl.values?.[0] const secondVal = secondEl.values?.[0] @@ -36,9 +36,9 @@ export const TrialTable: FC<{ return 0 } if (firstVal === undefined) { - return -1 + return ascending ? -1 : 1 } else if (secondVal === undefined) { - return 1 + return ascending ? 1 : -1 } if (firstVal === "-inf" || secondVal === "inf") { return 1 @@ -63,7 +63,7 @@ export const TrialTable: FC<{ ? objectiveNames[objectiveId] : `Objective ${objectiveId}`, sortable: true, - less: (firstEl, secondEl): number => { + less: (firstEl, secondEl, ascending): number => { const firstVal = firstEl.values?.[objectiveId] const secondVal = secondEl.values?.[objectiveId] @@ -71,9 +71,9 @@ export const TrialTable: FC<{ return 0 } if (firstVal === undefined) { - return -1 + return ascending ? -1 : 1 } else if (secondVal === undefined) { - return 1 + return ascending ? 1 : -1 } if (firstVal === "-inf" || secondVal === "inf") { return 1 @@ -106,7 +106,7 @@ export const TrialTable: FC<{ ?.param_external_value || null, sortable: sortable, filterable: filterable, - less: (firstEl, secondEl): number => { + less: (firstEl, secondEl, _): number => { const firstVal = firstEl.params.find( (p) => p.name === s.name )?.param_internal_value @@ -146,7 +146,7 @@ export const TrialTable: FC<{ ?.value || null, sortable: attr_spec.sortable, filterable: !attr_spec.sortable, - less: (firstEl, secondEl): number => { + less: (firstEl, secondEl, _): number => { const firstVal = firstEl.user_attrs.find( (attr) => attr.key === attr_spec.key )?.value diff --git a/standalone_app/src/components/DataGrid.tsx b/standalone_app/src/components/DataGrid.tsx index 2d6a5670..d19ec9d9 100644 --- a/standalone_app/src/components/DataGrid.tsx +++ b/standalone_app/src/components/DataGrid.tsx @@ -28,7 +28,7 @@ interface DataGridColumn { field: keyof T label: string sortable?: boolean - less?: (a: T, b: T) => number + less?: (a: T, b: T, ascending: boolean) => number filterable?: boolean toCellValue?: (rowIndex: number) => string | React.ReactNode padding?: "normal" | "checkbox" | "none" @@ -358,7 +358,8 @@ function stableSort( const stabilizedThis = array.map((el, index) => [el, index] as [T, number]) stabilizedThis.sort((a, b) => { if (less) { - const result = order == "asc" ? -less(a[0], b[0]) : less(a[0], b[0]) + const ascending = order == "asc" + const result = ascending ? -less(a[0], b[0], ascending) : less(a[0], b[0], ascending) if (result !== 0) return result } else { const result = comparator(a[0], b[0]) diff --git a/standalone_app/src/components/TrialTable.tsx b/standalone_app/src/components/TrialTable.tsx index 58d2d101..9e890c83 100644 --- a/standalone_app/src/components/TrialTable.tsx +++ b/standalone_app/src/components/TrialTable.tsx @@ -25,7 +25,7 @@ export const TrialTable: FC<{ field: "values", label: "Value", sortable: true, - less: (firstEl, secondEl): number => { + less: (firstEl, secondEl, ascending): number => { const firstVal = firstEl.values?.[0] const secondVal = secondEl.values?.[0] @@ -33,9 +33,9 @@ export const TrialTable: FC<{ return 0 } if (firstVal === undefined) { - return -1 + return ascending ? -1 : 1 } else if (secondVal === undefined) { - return 1 + return ascending ? 1 : -1 } if (firstVal === "-inf" || secondVal === "inf") { return 1 @@ -57,7 +57,7 @@ export const TrialTable: FC<{ field: "values", label: `Objective ${objectiveId}`, sortable: true, - less: (firstEl, secondEl): number => { + less: (firstEl, secondEl, ascending): number => { const firstVal = firstEl.values?.[objectiveId] const secondVal = secondEl.values?.[objectiveId] @@ -65,9 +65,9 @@ export const TrialTable: FC<{ return 0 } if (firstVal === undefined) { - return -1 + return ascending ? -1 : 1 } else if (secondVal === undefined) { - return 1 + return ascending ? 1 : -1 } if (firstVal === "-inf" || secondVal === "inf") { return 1 @@ -96,7 +96,7 @@ export const TrialTable: FC<{ null, sortable: true, filterable: false, - less: (firstEl, secondEl): number => { + less: (firstEl, secondEl, _): number => { const firstVal = firstEl.params.find( (p) => p.name === s.name )?.param_internal_value @@ -126,7 +126,7 @@ export const TrialTable: FC<{ ?.value || null, sortable: attr_spec.sortable, filterable: false, - less: (firstEl, secondEl): number => { + less: (firstEl, secondEl, _): number => { const firstVal = firstEl.user_attrs.find( (attr) => attr.key === attr_spec.key )?.value From a4ff5c61afd4e98cd94b0d94b5620940c07446c2 Mon Sep 17 00:00:00 2001 From: Victoria A <52001888+adjeiv@users.noreply.github.com> Date: Wed, 25 Oct 2023 08:14:02 +0100 Subject: [PATCH 08/26] Suppress no-unused-vars --- optuna_dashboard/ts/components/TrialTable.tsx | 2 ++ standalone_app/src/components/TrialTable.tsx | 2 ++ 2 files changed, 4 insertions(+) diff --git a/optuna_dashboard/ts/components/TrialTable.tsx b/optuna_dashboard/ts/components/TrialTable.tsx index 6090ee85..c708f6ee 100644 --- a/optuna_dashboard/ts/components/TrialTable.tsx +++ b/optuna_dashboard/ts/components/TrialTable.tsx @@ -106,6 +106,7 @@ export const TrialTable: FC<{ ?.param_external_value || null, sortable: sortable, filterable: filterable, + // eslint-disable-next-line @typescript-eslint/no-unused-vars less: (firstEl, secondEl, _): number => { const firstVal = firstEl.params.find( (p) => p.name === s.name @@ -146,6 +147,7 @@ export const TrialTable: FC<{ ?.value || null, sortable: attr_spec.sortable, filterable: !attr_spec.sortable, + // eslint-disable-next-line @typescript-eslint/no-unused-vars less: (firstEl, secondEl, _): number => { const firstVal = firstEl.user_attrs.find( (attr) => attr.key === attr_spec.key diff --git a/standalone_app/src/components/TrialTable.tsx b/standalone_app/src/components/TrialTable.tsx index 9e890c83..5a0efc96 100644 --- a/standalone_app/src/components/TrialTable.tsx +++ b/standalone_app/src/components/TrialTable.tsx @@ -96,6 +96,7 @@ export const TrialTable: FC<{ null, sortable: true, filterable: false, + // eslint-disable-next-line @typescript-eslint/no-unused-vars less: (firstEl, secondEl, _): number => { const firstVal = firstEl.params.find( (p) => p.name === s.name @@ -126,6 +127,7 @@ export const TrialTable: FC<{ ?.value || null, sortable: attr_spec.sortable, filterable: false, + // eslint-disable-next-line @typescript-eslint/no-unused-vars less: (firstEl, secondEl, _): number => { const firstVal = firstEl.user_attrs.find( (attr) => attr.key === attr_spec.key From 4719e7df98feb7ea08dabf871ef469936c0e7872 Mon Sep 17 00:00:00 2001 From: HideakiImamura Date: Wed, 25 Oct 2023 18:13:04 +0900 Subject: [PATCH 09/26] Follow review comments --- optuna_dashboard/ts/components/GraphRank.tsx | 85 ++++++++++---------- 1 file changed, 42 insertions(+), 43 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphRank.tsx b/optuna_dashboard/ts/components/GraphRank.tsx index b6e2fd1f..553612de 100644 --- a/optuna_dashboard/ts/components/GraphRank.tsx +++ b/optuna_dashboard/ts/components/GraphRank.tsx @@ -21,8 +21,8 @@ const plotDomId = "graph-rank" interface AxisInfo { name: string range: [number, number] - is_log: boolean - is_cat: boolean + isLog: boolean + isCat: boolean } interface RankPlotInfo { @@ -45,10 +45,10 @@ export const GraphRank: FC<{ const [yParam, setYParam] = useState(null) const objectiveNames: string[] = study?.objective_names || [] - if (xParam == null && searchSpace.length > 0) { + if (xParam === null && searchSpace.length > 0) { setXParam(searchSpace[0]) } - if (yParam == null && searchSpace.length > 1) { + if (yParam === null && searchSpace.length > 1) { setYParam(searchSpace[1]) } @@ -64,13 +64,12 @@ export const GraphRank: FC<{ setYParam(param || null) } - const rankPlotInfo = getRankPlotInfo(study, objectiveId, xParam, yParam) - useEffect(() => { if (study != null) { + const rankPlotInfo = getRankPlotInfo(study, objectiveId, xParam, yParam) plotRank(rankPlotInfo, theme.palette.mode) } - }, [study, theme.palette.mode]) + }, [study, objectiveId, xParam, yParam, theme.palette.mode]) const space: SearchSpaceItem[] = study ? study.union_search_space : [] @@ -146,19 +145,19 @@ const getRankPlotInfo = ( } const trials = study.trials - const filtered_trials = trials.filter(filterFunc) - if (filtered_trials.length < 2 || xParam == null || yParam == null) { + const filteredTrials = trials.filter(filterFunc) + if (filteredTrials.length < 2 || xParam === null || yParam === null) { return null } - const xAxis = getAxisInfo(filtered_trials, xParam) - const yAxis = getAxisInfo(filtered_trials, yParam) + const xAxis = getAxisInfo(filteredTrials, xParam) + const yAxis = getAxisInfo(filteredTrials, yParam) - const xValues: number[] = [] - const yValues: number[] = [] + const xValues: (string | number)[] = [] + const yValues: (string | number)[] = [] const zValues: number[] = [] const hovertext: string[] = [] - filtered_trials.forEach((trial) => { + filteredTrials.forEach((trial) => { const xValue = trial.params.find((p) => p.name === xAxis.name)?.param_internal_value || null @@ -218,8 +217,8 @@ const getAxisInfoForCategorical = ( return { name: param, range: [min, max], - is_log: false, - is_cat: true, + isLog: false, + isCat: true, } } @@ -232,14 +231,14 @@ const getAxisInfoForNumerical = ( (trial) => trial.params.find((p) => p.name === param)?.param_internal_value || null ) - const non_null_values: number[] = [] + const nonNullValues: number[] = [] values.forEach((value) => { if (value !== null) { - non_null_values.push(value) + nonNullValues.push(value) } }) - let min = Math.min(...non_null_values) - let max = Math.max(...non_null_values) + let min = Math.min(...nonNullValues) + let max = Math.max(...nonNullValues) if (distribution.log) { const padding = (Math.log10(max) - Math.log10(min)) * PADDING_RATIO min = Math.pow(10, Math.log10(min) - padding) @@ -253,58 +252,58 @@ const getAxisInfoForNumerical = ( return { name: param, range: [min, max], - is_log: distribution.log, - is_cat: false, + isLog: distribution.log, + isCat: false, } } const getColors = (values: number[]): number[] => { - const raw_ranks = getOrderWithSameOrderAveraging(values) - let color_idxs: number[] = [] + const rawRanks = getOrderWithSameOrderAveraging(values) + let colorIdxs: number[] = [] if (values.length > 2) { - color_idxs = raw_ranks.map((rank) => rank / (values.length - 1)) + colorIdxs = rawRanks.map((rank) => rank / (values.length - 1)) } else { - color_idxs = [0.5] + colorIdxs = [0.5] } - return color_idxs + return colorIdxs } const getOrderWithSameOrderAveraging = (values: number[]): number[] => { - const sorted_values = values.slice().sort() + const sortedValues = values.slice().sort() const ranks: number[] = [] values.forEach((value) => { - const first_index = sorted_values.indexOf(value) - const last_index = sorted_values.lastIndexOf(value) - const sum_of_the_value = - ((first_index + last_index) * (last_index - first_index + 1)) / 2 - const rank = sum_of_the_value / (last_index - first_index + 1) + const firstIndex = sortedValues.indexOf(value) + const lastIndex = sortedValues.lastIndexOf(value) + const sumOfTheValue = + ((firstIndex + lastIndex) * (lastIndex - firstIndex + 1)) / 2 + const rank = sumOfTheValue / (lastIndex - firstIndex + 1) ranks.push(rank) }) return ranks } -const plotRank = (rank_plot_info: RankPlotInfo | null, mode: string) => { +const plotRank = (rankPlotInfo: RankPlotInfo | null, mode: string) => { if (document.getElementById(plotDomId) === null) { return } - if (rank_plot_info === null) { + if (rankPlotInfo === null) { plotly.react(plotDomId, [], { template: mode === "dark" ? plotlyDarkTemplate : {}, }) return } - const xAxis = rank_plot_info.xaxis - const yAxis = rank_plot_info.yaxis + const xAxis = rankPlotInfo.xaxis + const yAxis = rankPlotInfo.yaxis const layout: Partial = { xaxis: { title: xAxis.name, - type: xAxis.is_cat ? "category" : xAxis.is_log ? "log" : "linear", + type: xAxis.isCat ? "category" : xAxis.isLog ? "log" : "linear", }, yaxis: { title: yAxis.name, - type: yAxis.is_cat ? "category" : yAxis.is_log ? "log" : "linear", + type: yAxis.isCat ? "category" : yAxis.isLog ? "log" : "linear", }, margin: { l: 50, @@ -318,10 +317,10 @@ const plotRank = (rank_plot_info: RankPlotInfo | null, mode: string) => { const plotData: Partial[] = [ { type: "scatter", - x: rank_plot_info.xvalues, - y: rank_plot_info.yvalues, + x: rankPlotInfo.xvalues, + y: rankPlotInfo.yvalues, marker: { - color: rank_plot_info.colors, + color: rankPlotInfo.colors, colorscale: "Portland", colorbar: { title: "Rank", @@ -335,7 +334,7 @@ const plotRank = (rank_plot_info: RankPlotInfo | null, mode: string) => { mode: "markers", showlegend: false, hovertemplate: "%{hovertext}", - hovertext: rank_plot_info.hovertext, + hovertext: rankPlotInfo.hovertext, }, ] plotly.react(plotDomId, plotData, layout) From 29a5833cd02d5edb1c4699539688058b1bbb0f57 Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 26 Oct 2023 15:40:19 +0900 Subject: [PATCH 10/26] Fix broken tutorial links --- docs/tutorials/preferential-optimization.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/preferential-optimization.rst b/docs/tutorials/preferential-optimization.rst index a192e883..ae662383 100644 --- a/docs/tutorials/preferential-optimization.rst +++ b/docs/tutorials/preferential-optimization.rst @@ -5,11 +5,11 @@ What is Preferential Optimization? ---------------------------------- Preferential optimization is a method for optimizing hyperparameters, focusing of human preferences, by determining which trial is superior when comparing a pair. -It differs from `human-in-the-loop optimization utilizing objective form widgets `_, +It differs from :ref:`human-in-the-loop optimization utilizing objective form widgets `, which relies on absolute evaluations, as it significantly reduces fluctuations in evaluators' criteria, thus ensuring more consistent results. In this tutorial, we'll interactively optimize RGB values to generate a color resembling a "sunset hue", -aligining with the problem setting in `this tutorial `_. +aligining with the problem setting in :ref:`this tutorial `. Familiarity with the tutorial ob objective form widgets may enhance your understanding. How to Run Preferential Optimization From 31360a4597863da09019edbbfbec6060e8c584c1 Mon Sep 17 00:00:00 2001 From: Victoria A <52001888+adjeiv@users.noreply.github.com> Date: Fri, 27 Oct 2023 08:21:49 +0100 Subject: [PATCH 11/26] Lint --- optuna_dashboard/ts/components/DataGrid.tsx | 4 +++- standalone_app/src/components/DataGrid.tsx | 4 +++- standalone_app/src/components/TrialTable.tsx | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/optuna_dashboard/ts/components/DataGrid.tsx b/optuna_dashboard/ts/components/DataGrid.tsx index d19ec9d9..24e45dc7 100644 --- a/optuna_dashboard/ts/components/DataGrid.tsx +++ b/optuna_dashboard/ts/components/DataGrid.tsx @@ -359,7 +359,9 @@ function stableSort( stabilizedThis.sort((a, b) => { if (less) { const ascending = order == "asc" - const result = ascending ? -less(a[0], b[0], ascending) : less(a[0], b[0], ascending) + const result = ascending + ? -less(a[0], b[0], ascending) + : less(a[0], b[0], ascending) if (result !== 0) return result } else { const result = comparator(a[0], b[0]) diff --git a/standalone_app/src/components/DataGrid.tsx b/standalone_app/src/components/DataGrid.tsx index d19ec9d9..24e45dc7 100644 --- a/standalone_app/src/components/DataGrid.tsx +++ b/standalone_app/src/components/DataGrid.tsx @@ -359,7 +359,9 @@ function stableSort( stabilizedThis.sort((a, b) => { if (less) { const ascending = order == "asc" - const result = ascending ? -less(a[0], b[0], ascending) : less(a[0], b[0], ascending) + const result = ascending + ? -less(a[0], b[0], ascending) + : less(a[0], b[0], ascending) if (result !== 0) return result } else { const result = comparator(a[0], b[0]) diff --git a/standalone_app/src/components/TrialTable.tsx b/standalone_app/src/components/TrialTable.tsx index 5a0efc96..97a04f3c 100644 --- a/standalone_app/src/components/TrialTable.tsx +++ b/standalone_app/src/components/TrialTable.tsx @@ -65,7 +65,7 @@ export const TrialTable: FC<{ return 0 } if (firstVal === undefined) { - return ascending ? -1 : 1 + return ascending ? -1 : 1 } else if (secondVal === undefined) { return ascending ? 1 : -1 } From 1c7645dfaf0afb13040f0bef0a73499d99833498 Mon Sep 17 00:00:00 2001 From: HideakiImamura Date: Tue, 31 Oct 2023 14:45:40 +0900 Subject: [PATCH 12/26] Fix for categorical variables --- optuna_dashboard/ts/components/GraphRank.tsx | 39 ++++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphRank.tsx b/optuna_dashboard/ts/components/GraphRank.tsx index 553612de..92cd8dff 100644 --- a/optuna_dashboard/ts/components/GraphRank.tsx +++ b/optuna_dashboard/ts/components/GraphRank.tsx @@ -159,10 +159,10 @@ const getRankPlotInfo = ( const hovertext: string[] = [] filteredTrials.forEach((trial) => { const xValue = - trial.params.find((p) => p.name === xAxis.name)?.param_internal_value || + trial.params.find((p) => p.name === xAxis.name)?.param_external_value || null const yValue = - trial.params.find((p) => p.name === yAxis.name)?.param_internal_value || + trial.params.find((p) => p.name === yAxis.name)?.param_external_value || null if (trial.values === undefined || xValue === null || yValue === null) { return @@ -314,11 +314,42 @@ const plotRank = (rankPlotInfo: RankPlotInfo | null, mode: string) => { uirevision: "true", template: mode === "dark" ? plotlyDarkTemplate : {}, } + + let xValues = rankPlotInfo.xvalues + let yValues = rankPlotInfo.yvalues + if (xAxis.isCat && !yAxis.isCat) { + const xIndices: number[] = Array + .from(Array(xValues.length).keys()) + .sort((a, b) => xValues[a].toString().toLowerCase().localeCompare(xValues[b].toString().toLowerCase())) + xValues = xIndices.map((i) => xValues[i]) + yValues = xIndices.map((i) => yValues[i]) + } + if (!xAxis.isCat && yAxis.isCat) { + const yIndices: number[] = Array + .from(Array(yValues.length).keys()) + .sort((a, b) => yValues[a].toString().toLowerCase().localeCompare(yValues[b].toString().toLowerCase())) + xValues = yIndices.map((i) => xValues[i]) + yValues = yIndices.map((i) => yValues[i]) + } + if (xAxis.isCat && yAxis.isCat) { + const indices: number[] = Array + .from(Array(xValues.length).keys()) + .sort((a, b) => { + const xComp = xValues[a].toString().toLowerCase().localeCompare(xValues[b].toString().toLowerCase()) + if (xComp !== 0) { + return xComp + } + return yValues[a].toString().toLowerCase().localeCompare(yValues[b].toString().toLowerCase()) + }) + xValues = indices.map((i) => xValues[i]) + yValues = indices.map((i) => yValues[i]) + } + const plotData: Partial[] = [ { type: "scatter", - x: rankPlotInfo.xvalues, - y: rankPlotInfo.yvalues, + x: xValues, + y: yValues, marker: { color: rankPlotInfo.colors, colorscale: "Portland", From 8c02487afba39620fc616a70cebf6ea339638e66 Mon Sep 17 00:00:00 2001 From: HideakiImamura Date: Tue, 31 Oct 2023 16:32:07 +0900 Subject: [PATCH 13/26] Applt formatter --- optuna_dashboard/ts/components/GraphRank.tsx | 40 +++++++++++++------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphRank.tsx b/optuna_dashboard/ts/components/GraphRank.tsx index 92cd8dff..4e3e7544 100644 --- a/optuna_dashboard/ts/components/GraphRank.tsx +++ b/optuna_dashboard/ts/components/GraphRank.tsx @@ -314,33 +314,47 @@ const plotRank = (rankPlotInfo: RankPlotInfo | null, mode: string) => { uirevision: "true", template: mode === "dark" ? plotlyDarkTemplate : {}, } - + let xValues = rankPlotInfo.xvalues let yValues = rankPlotInfo.yvalues if (xAxis.isCat && !yAxis.isCat) { - const xIndices: number[] = Array - .from(Array(xValues.length).keys()) - .sort((a, b) => xValues[a].toString().toLowerCase().localeCompare(xValues[b].toString().toLowerCase())) + const xIndices: number[] = Array.from(Array(xValues.length).keys()).sort( + (a, b) => + xValues[a] + .toString() + .toLowerCase() + .localeCompare(xValues[b].toString().toLowerCase()) + ) xValues = xIndices.map((i) => xValues[i]) yValues = xIndices.map((i) => yValues[i]) } if (!xAxis.isCat && yAxis.isCat) { - const yIndices: number[] = Array - .from(Array(yValues.length).keys()) - .sort((a, b) => yValues[a].toString().toLowerCase().localeCompare(yValues[b].toString().toLowerCase())) + const yIndices: number[] = Array.from(Array(yValues.length).keys()).sort( + (a, b) => + yValues[a] + .toString() + .toLowerCase() + .localeCompare(yValues[b].toString().toLowerCase()) + ) xValues = yIndices.map((i) => xValues[i]) yValues = yIndices.map((i) => yValues[i]) } if (xAxis.isCat && yAxis.isCat) { - const indices: number[] = Array - .from(Array(xValues.length).keys()) - .sort((a, b) => { - const xComp = xValues[a].toString().toLowerCase().localeCompare(xValues[b].toString().toLowerCase()) + const indices: number[] = Array.from(Array(xValues.length).keys()).sort( + (a, b) => { + const xComp = xValues[a] + .toString() + .toLowerCase() + .localeCompare(xValues[b].toString().toLowerCase()) if (xComp !== 0) { return xComp } - return yValues[a].toString().toLowerCase().localeCompare(yValues[b].toString().toLowerCase()) - }) + return yValues[a] + .toString() + .toLowerCase() + .localeCompare(yValues[b].toString().toLowerCase()) + } + ) xValues = indices.map((i) => xValues[i]) yValues = indices.map((i) => yValues[i]) } From 0c644462ce6b427f561b216106d70b7d9d6f178b Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Tue, 31 Oct 2023 08:35:55 +0100 Subject: [PATCH 14/26] [feat] Support intermediate value plots for constrained optimization --- .../ts/components/GraphIntermediateValues.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx index ca0bfda0..13412b87 100644 --- a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx +++ b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx @@ -84,16 +84,21 @@ const plotIntermediateValue = ( const values = trial.intermediate_values.filter( (iv) => iv.value !== "inf" && iv.value !== "-inf" && iv.value !== "nan" ) + const isFeasible = trial.constraints.every((c) => c <= 0) + let name = `trial #${trial.number}` + if (trial.state === "Running") { + name += ` (running)` + } else { + name += isFeasible ? `` : ` (infeasible)` + } return { x: values.map((iv) => iv.step), y: values.map((iv) => iv.value), marker: { maxdisplayed: 10 }, mode: "lines+markers", type: "scatter", - name: - trial.state !== "Running" - ? `trial #${trial.number}` - : `trial #${trial.number} (running)`, + name, + ...(isFeasible && { line: { color: "#CCCCCC" } }), } }) plotly.react(plotDomId, plotData, layout) From 26329560928fcad588c4eb337c2c6eaad430c157 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Tue, 31 Oct 2023 08:53:44 +0100 Subject: [PATCH 15/26] [feat] Make the infeasible plots dashed --- optuna_dashboard/ts/components/GraphIntermediateValues.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx index 13412b87..a4099c57 100644 --- a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx +++ b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx @@ -98,7 +98,7 @@ const plotIntermediateValue = ( mode: "lines+markers", type: "scatter", name, - ...(isFeasible && { line: { color: "#CCCCCC" } }), + ...(isFeasible && { line: { color: "#CCCCCC", dash: "dash" } }), } }) plotly.react(plotDomId, plotData, layout) From 0e1feb3cf6f51e0dd5360ff108bff643080d222f Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Tue, 31 Oct 2023 11:08:28 +0100 Subject: [PATCH 16/26] Fix an error in infeasible intermediate value plot --- optuna_dashboard/ts/components/GraphIntermediateValues.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx index a4099c57..9ee4a36d 100644 --- a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx +++ b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx @@ -98,7 +98,7 @@ const plotIntermediateValue = ( mode: "lines+markers", type: "scatter", name, - ...(isFeasible && { line: { color: "#CCCCCC", dash: "dash" } }), + ...(!isFeasible && { line: { color: "#CCCCCC", dash: "dash" } }), } }) plotly.react(plotDomId, plotData, layout) From 00606bc3f5118b6392c3b4140693928ed9f21cb5 Mon Sep 17 00:00:00 2001 From: Toshihiko Yanase Date: Wed, 1 Nov 2023 18:27:54 +0900 Subject: [PATCH 17/26] Add constraints handling for contour plot. --- optuna_dashboard/ts/components/GraphContour.tsx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphContour.tsx b/optuna_dashboard/ts/components/GraphContour.tsx index b186fffe..a5416f5b 100644 --- a/optuna_dashboard/ts/components/GraphContour.tsx +++ b/optuna_dashboard/ts/components/GraphContour.tsx @@ -195,12 +195,16 @@ const plotContour = ( const xValues: plotly.Datum[] = [] const yValues: plotly.Datum[] = [] const zValues: plotly.Datum[][] = new Array(yIndices.length) + const feasibleXY = new Set() for (let j = 0; j < yIndices.length; j++) { zValues[j] = new Array(xIndices.length).fill(null) } filteredTrials.forEach((trial, i) => { if (xAxis.values[i] && yAxis.values[i] && trial.values) { + if (trial.constraints.every((c) => c <= 0)) { + feasibleXY.add(xValues.length) + } const xValue = xAxis.values[i] as string | number const yValue = yAxis.values[i] as string | number xValues.push(xValue) @@ -234,12 +238,20 @@ const plotContour = ( }, { type: "scatter", - x: xValues, - y: yValues, + x: xValues.filter((_, i) => feasibleXY.has(i)), + y: yValues.filter((_, i) => feasibleXY.has(i)), marker: { line: { width: 2.0, color: "Grey" }, color: "black" }, mode: "markers", showlegend: false, }, + { + type: "scatter", + x: xValues.filter((_, i) => !feasibleXY.has(i)), + y: yValues.filter((_, i) => !feasibleXY.has(i)), + marker: { line: { width: 2.0, color: "Grey" }, color: "#cccccc" }, + mode: "markers", + showlegend: false, + }, ] plotly.react(plotDomId, plotData, layout) return From 79b50cc5005eb81b841cfafc6f483f7e4251f896 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Thu, 2 Nov 2023 04:56:25 +0100 Subject: [PATCH 18/26] Address the mamu's comment --- .../ts/components/GraphIntermediateValues.tsx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx index 9ee4a36d..7ccce637 100644 --- a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx +++ b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx @@ -85,12 +85,13 @@ const plotIntermediateValue = ( (iv) => iv.value !== "inf" && iv.value !== "-inf" && iv.value !== "nan" ) const isFeasible = trial.constraints.every((c) => c <= 0) - let name = `trial #${trial.number}` - if (trial.state === "Running") { - name += ` (running)` - } else { - name += isFeasible ? `` : ` (infeasible)` - } + const name = `trial #${trial.number} ${ + trial.state === "Running" + ? "(running)" + : !isFeasible + ? " (infeasible)" + : "" + }` return { x: values.map((iv) => iv.step), y: values.map((iv) => iv.value), From 634f959d9039181a4a104a8d00245508e5d6a079 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Thu, 2 Nov 2023 05:00:39 +0100 Subject: [PATCH 19/26] Make it close to the original --- .../ts/components/GraphIntermediateValues.tsx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx index 7ccce637..fc2be23e 100644 --- a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx +++ b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx @@ -85,20 +85,19 @@ const plotIntermediateValue = ( (iv) => iv.value !== "inf" && iv.value !== "-inf" && iv.value !== "nan" ) const isFeasible = trial.constraints.every((c) => c <= 0) - const name = `trial #${trial.number} ${ - trial.state === "Running" - ? "(running)" - : !isFeasible - ? " (infeasible)" - : "" - }` return { x: values.map((iv) => iv.step), y: values.map((iv) => iv.value), marker: { maxdisplayed: 10 }, mode: "lines+markers", type: "scatter", - name, + name: `trial #${trial.number} ${ + trial.state === "Running" + ? "(running)" + : !isFeasible + ? "(infeasible)" + : "" + }`, ...(!isFeasible && { line: { color: "#CCCCCC", dash: "dash" } }), } }) From 2403246749fa0e4e882c968e265ef7af5204d6a9 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Thu, 2 Nov 2023 06:33:34 +0100 Subject: [PATCH 20/26] Remove dashed lines for now --- optuna_dashboard/ts/components/GraphIntermediateValues.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx index fc2be23e..2d86e464 100644 --- a/optuna_dashboard/ts/components/GraphIntermediateValues.tsx +++ b/optuna_dashboard/ts/components/GraphIntermediateValues.tsx @@ -98,7 +98,7 @@ const plotIntermediateValue = ( ? "(infeasible)" : "" }`, - ...(!isFeasible && { line: { color: "#CCCCCC", dash: "dash" } }), + ...(!isFeasible && { line: { color: "#CCCCCC" } }), } }) plotly.react(plotDomId, plotData, layout) From 102c910bf76cdd8de7461ae7418b73f7190268c2 Mon Sep 17 00:00:00 2001 From: Kenshin Abe Date: Tue, 7 Nov 2023 01:31:00 +0900 Subject: [PATCH 21/26] Unify duplicated logic in plotSlice --- optuna_dashboard/ts/components/GraphSlice.tsx | 61 ++++++------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphSlice.tsx b/optuna_dashboard/ts/components/GraphSlice.tsx index f37a5413..1498f451 100644 --- a/optuna_dashboard/ts/components/GraphSlice.tsx +++ b/optuna_dashboard/ts/components/GraphSlice.tsx @@ -196,57 +196,36 @@ const plotSlice = ( ) const trialNumbers: number[] = trials.map((t) => t.number) - if (selectedParamSpace.distribution.type !== "CategoricalDistribution") { - const trace: plotly.Data[] = [ - { - type: "scatter", - x: values, - y: objectiveValues, - mode: "markers", - marker: { - color: trialNumbers, - colorscale: "Blues", - reversescale: true, - colorbar: { - title: "Trial", - }, - line: { - color: "Grey", - width: 0.5, - }, + const trace: plotly.Data[] = [ + { + type: "scatter", + x: values, + y: objectiveValues, + mode: "markers", + marker: { + color: trialNumbers, + colorscale: "Blues", + reversescale: true, + colorbar: { + title: "Trial", + }, + line: { + color: "Grey", + width: 0.5, }, }, - ] + }, + ] + if (selectedParamSpace.distribution.type !== "CategoricalDistribution") { layout["xaxis"] = { title: selectedParamTarget.toLabel(), type: isLogScale(selectedParamSpace) ? "log" : "linear", gridwidth: 1, automargin: true, // Otherwise the label is outside of the plot } - plotly.react(plotDomId, trace, layout) } else { const vocabArr = selectedParamSpace.distribution.choices.map((c) => c.value) const tickvals: number[] = vocabArr.map((v, i) => i) - const trace: plotly.Data[] = [ - { - type: "scatter", - x: values, - y: objectiveValues, - mode: "markers", - marker: { - color: trialNumbers, - colorscale: "Blues", - reversescale: true, - colorbar: { - title: "Trial", - }, - line: { - color: "Grey", - width: 0.5, - }, - }, - }, - ] layout["xaxis"] = { title: selectedParamTarget.toLabel(), type: "linear", @@ -255,6 +234,6 @@ const plotSlice = ( ticktext: vocabArr, automargin: true, // Otherwise the label is outside of the plot } - plotly.react(plotDomId, trace, layout) } + plotly.react(plotDomId, trace, layout) } From 2c50b5afeaa45bc9fff256a2c553f74547520e31 Mon Sep 17 00:00:00 2001 From: Toshihiko Yanase Date: Wed, 8 Nov 2023 11:45:14 +0900 Subject: [PATCH 22/26] Pin Python version to 3.11 --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 69923d04..60dd6ea0 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -58,7 +58,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: '3.11' architecture: x64 - name: Install dependencies run: | From 5076818e46160dca255bcbd2453a5f1dd28261c3 Mon Sep 17 00:00:00 2001 From: Kenshin Abe Date: Wed, 8 Nov 2023 18:42:41 +0900 Subject: [PATCH 23/26] Support constraints in slice plot --- optuna_dashboard/ts/components/GraphSlice.tsx | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphSlice.tsx b/optuna_dashboard/ts/components/GraphSlice.tsx index 1498f451..2a807cb7 100644 --- a/optuna_dashboard/ts/components/GraphSlice.tsx +++ b/optuna_dashboard/ts/components/GraphSlice.tsx @@ -188,22 +188,38 @@ const plotSlice = ( return } - const objectiveValues: number[] = trials.map( + const feasibleTrials: Trial[] = [] + const infeasibleTrials: Trial[] = [] + trials.forEach((t) => { + if (t.constraints.every((c) => c <= 0)) { + feasibleTrials.push(t) + } else { + infeasibleTrials.push(t) + } + }) + + const feasibleObjectiveValues: number[] = feasibleTrials.map( (t) => objectiveTarget.getTargetValue(t) as number ) - const values = trials.map( - (t) => selectedParamTarget.getTargetValue(t) as number + const infeasibleObjectiveValues: number[] = infeasibleTrials.map( + (t) => objectiveTarget.getTargetValue(t) as number ) - const trialNumbers: number[] = trials.map((t) => t.number) + const feasibleValues = feasibleTrials.map( + (t) => selectedParamTarget.getTargetValue(t) as number + ) + const infeasibleValues = infeasibleTrials.map( + (t) => selectedParamTarget.getTargetValue(t) as number + ) const trace: plotly.Data[] = [ { type: "scatter", - x: values, - y: objectiveValues, + x: feasibleValues, + y: feasibleObjectiveValues, mode: "markers", + name: "Feasible Trial", marker: { - color: trialNumbers, + color: feasibleTrials.map((t) => t.number), colorscale: "Blues", reversescale: true, colorbar: { @@ -215,6 +231,17 @@ const plotSlice = ( }, }, }, + { + type: "scatter", + x: infeasibleValues, + y: infeasibleObjectiveValues, + mode: "markers", + name: "Infeasible Trial", + marker: { + color: "#cccccc", + reversescale: true, + }, + }, ] if (selectedParamSpace.distribution.type !== "CategoricalDistribution") { layout["xaxis"] = { From a53d7a266c33de35e4c59ffac82ab15e55008b64 Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Thu, 9 Nov 2023 13:49:33 +0900 Subject: [PATCH 24/26] Update GraphRank.tsx --- optuna_dashboard/ts/components/GraphRank.tsx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/optuna_dashboard/ts/components/GraphRank.tsx b/optuna_dashboard/ts/components/GraphRank.tsx index 4e3e7544..9763ac17 100644 --- a/optuna_dashboard/ts/components/GraphRank.tsx +++ b/optuna_dashboard/ts/components/GraphRank.tsx @@ -32,6 +32,7 @@ interface RankPlotInfo { yvalues: (string | number)[] zvalues: number[] colors: number[] + is_feasible: boolean[] hovertext: string[] } @@ -156,6 +157,7 @@ const getRankPlotInfo = ( const xValues: (string | number)[] = [] const yValues: (string | number)[] = [] const zValues: number[] = [] + const isFeasible: boolean[] = [] const hovertext: string[] = [] filteredTrials.forEach((trial) => { const xValue = @@ -168,9 +170,11 @@ const getRankPlotInfo = ( return } const zValue = Number(trial.values[objectiveId]) + const feasibility = trial.constraints.every((c) => c <= 0) xValues.push(xValue) yValues.push(yValue) zValues.push(zValue) + isFeasible.push(feasibility) hovertext.push(makeHovertext(trial)) }) @@ -183,6 +187,7 @@ const getRankPlotInfo = ( yvalues: yValues, zvalues: zValues, colors, + is_feasible: isFeasible, hovertext, } } @@ -382,5 +387,10 @@ const plotRank = (rankPlotInfo: RankPlotInfo | null, mode: string) => { hovertext: rankPlotInfo.hovertext, }, ] + for (let i = 0; i < plotData.length; i++) { + if(rankPlotInfo.is_feasible[i] == false){ + plotData[i].marker.color = "#cccccc" + } + } plotly.react(plotDomId, plotData, layout) } From 00ce7ceb00ca2b581d33e12c6f358b2ab80f2528 Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Thu, 9 Nov 2023 15:53:05 +0900 Subject: [PATCH 25/26] Update GraphRank.tsx --- optuna_dashboard/ts/components/GraphRank.tsx | 32 ++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphRank.tsx b/optuna_dashboard/ts/components/GraphRank.tsx index 9763ac17..662135d2 100644 --- a/optuna_dashboard/ts/components/GraphRank.tsx +++ b/optuna_dashboard/ts/components/GraphRank.tsx @@ -367,10 +367,10 @@ const plotRank = (rankPlotInfo: RankPlotInfo | null, mode: string) => { const plotData: Partial[] = [ { type: "scatter", - x: xValues, - y: yValues, + x: xValues.filter((_, i) => rankPlotInfo.is_feasible[i]), + y: yValues.filter((_, i) => rankPlotInfo.is_feasible[i]), marker: { - color: rankPlotInfo.colors, + color: rankPlotInfo.colors.filter((_, i) => rankPlotInfo.is_feasible[i]), colorscale: "Portland", colorbar: { title: "Rank", @@ -382,15 +382,29 @@ const plotRank = (rankPlotInfo: RankPlotInfo | null, mode: string) => { }, }, mode: "markers", + name: "Feasible Trial", showlegend: false, hovertemplate: "%{hovertext}", - hovertext: rankPlotInfo.hovertext, + hovertext: rankPlotInfo.hovertext.filter((_, i) => rankPlotInfo.is_feasible[i]), }, - ] - for (let i = 0; i < plotData.length; i++) { - if(rankPlotInfo.is_feasible[i] == false){ - plotData[i].marker.color = "#cccccc" + { + type: "scatter", + x: xValues.filter((_, i) => !rankPlotInfo.is_feasible[i]), + y: yValues.filter((_, i) => !rankPlotInfo.is_feasible[i]), + marker: { + color: "#cccccc", + size: 10, + line: { + color: "Grey", + width: 0.5, + }, + }, + mode: "markers", + name: "Infeasible Trial", + showlegend: false, + hovertemplate: "%{hovertext}", + hovertext: rankPlotInfo.hovertext.filter((_, i) => !rankPlotInfo.is_feasible[i]), } - } + ] plotly.react(plotDomId, plotData, layout) } From 9182fba762c3dc367d07b5fec555ef4dda1a50d5 Mon Sep 17 00:00:00 2001 From: Hiroki Takizawa Date: Thu, 9 Nov 2023 17:13:46 +0900 Subject: [PATCH 26/26] fix lint --- optuna_dashboard/ts/components/GraphRank.tsx | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/optuna_dashboard/ts/components/GraphRank.tsx b/optuna_dashboard/ts/components/GraphRank.tsx index 662135d2..6781793f 100644 --- a/optuna_dashboard/ts/components/GraphRank.tsx +++ b/optuna_dashboard/ts/components/GraphRank.tsx @@ -370,7 +370,9 @@ const plotRank = (rankPlotInfo: RankPlotInfo | null, mode: string) => { x: xValues.filter((_, i) => rankPlotInfo.is_feasible[i]), y: yValues.filter((_, i) => rankPlotInfo.is_feasible[i]), marker: { - color: rankPlotInfo.colors.filter((_, i) => rankPlotInfo.is_feasible[i]), + color: rankPlotInfo.colors.filter( + (_, i) => rankPlotInfo.is_feasible[i] + ), colorscale: "Portland", colorbar: { title: "Rank", @@ -382,10 +384,11 @@ const plotRank = (rankPlotInfo: RankPlotInfo | null, mode: string) => { }, }, mode: "markers", - name: "Feasible Trial", showlegend: false, hovertemplate: "%{hovertext}", - hovertext: rankPlotInfo.hovertext.filter((_, i) => rankPlotInfo.is_feasible[i]), + hovertext: rankPlotInfo.hovertext.filter( + (_, i) => rankPlotInfo.is_feasible[i] + ), }, { type: "scatter", @@ -400,11 +403,12 @@ const plotRank = (rankPlotInfo: RankPlotInfo | null, mode: string) => { }, }, mode: "markers", - name: "Infeasible Trial", showlegend: false, hovertemplate: "%{hovertext}", - hovertext: rankPlotInfo.hovertext.filter((_, i) => !rankPlotInfo.is_feasible[i]), - } + hovertext: rankPlotInfo.hovertext.filter( + (_, i) => !rankPlotInfo.is_feasible[i] + ), + }, ] plotly.react(plotDomId, plotData, layout) }