diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index eaeecab8..ca49cc88 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -48,9 +48,11 @@ export const actionCreator = () => { const setArtifactIsAvailable = useSetRecoilState(artifactIsAvailable) const setStudyDetailState = (studyId: number, study: StudyDetail) => { - const newVal = Object.assign({}, studyDetails) - newVal[studyId] = study - setStudyDetails(newVal) + setStudyDetails((prevVal) => { + const newVal = Object.assign({}, prevVal) + newVal[studyId] = study + return newVal + }) } const setTrial = (studyId: number, trialIndex: number, trial: Trial) => { diff --git a/optuna_dashboard/ts/components/App.tsx b/optuna_dashboard/ts/components/App.tsx index f82b18d2..828108ef 100644 --- a/optuna_dashboard/ts/components/App.tsx +++ b/optuna_dashboard/ts/components/App.tsx @@ -12,6 +12,7 @@ import { CssBaseline, } from "@mui/material" +import { CompareStudies } from "./CompareStudies" import { StudyDetail } from "./StudyDetail" import { StudyList } from "./StudyList" import { StudyDetailBeta } from "./StudyDetailBeta" @@ -110,6 +111,12 @@ export const App: FC = () => { path={URL_PREFIX + "/studies/:studyId"} children={} /> + + } + /> } diff --git a/optuna_dashboard/ts/components/CompareStudies.tsx b/optuna_dashboard/ts/components/CompareStudies.tsx new file mode 100644 index 00000000..5e4d5f00 --- /dev/null +++ b/optuna_dashboard/ts/components/CompareStudies.tsx @@ -0,0 +1,339 @@ +import React, { ChangeEvent, FC, useEffect, useMemo, useState } from "react" +import { useRecoilValue } from "recoil" +import { useSnackbar } from "notistack" +import { Link } from "react-router-dom" +import { + Card, + CardContent, + FormControl, + Switch, + Typography, + Box, + useTheme, + IconButton, +} from "@mui/material" +import ChevronRightIcon from "@mui/icons-material/ChevronRight" +import Chip from "@mui/material/Chip" +import FormControlLabel from "@mui/material/FormControlLabel" +import Divider from "@mui/material/Divider" +import List from "@mui/material/List" +import ListItem from "@mui/material/ListItem" +import ListItemButton from "@mui/material/ListItemButton" +import ListItemText from "@mui/material/ListItemText" +import ListSubheader from "@mui/material/ListSubheader" +import HomeIcon from "@mui/icons-material/Home" + +import { actionCreator } from "../action" +import { studySummariesState, studyDetailsState } from "../state" +import { AppDrawer } from "./AppDrawer" +import { GraphEdfMultiStudies } from "./GraphEdf" +import { GraphHistoryMultiStudies } from "./GraphHistory" +import { useHistory, useLocation } from "react-router-dom" + +const useQuery = (): URLSearchParams => { + const { search } = useLocation() + return useMemo(() => new URLSearchParams(search), [search]) +} + +const useQueriedStudies = ( + studies: StudySummary[], + query: URLSearchParams +): StudySummary[] => { + return useMemo(() => { + const queried = query.get("ids") + if (queried === null) { + return [] + } + const ids = queried + .split(",") + .map((s) => parseInt(s)) + .filter((n) => !isNaN(n)) + return studies.filter((t) => ids.findIndex((n) => n === t.study_id) !== -1) + }, [studies, query]) +} + +const getStudyListLink = (ids: number[]): string => { + const base = URL_PREFIX + "/compare-studies" + if (ids.length > 0) { + return base + "?ids=" + ids.map((n) => n.toString()).join(",") + } + return base +} + +const isEqualDirections = ( + array1: StudyDirection[], + array2: StudyDirection[] +): boolean => { + let i = array1.length + if (i !== array2.length) return false + + while (i--) { + if (array1[i] !== array2[i]) return false + } + return true +} + +export const CompareStudies: FC<{ + toggleColorMode: () => void +}> = ({ toggleColorMode }) => { + const { enqueueSnackbar } = useSnackbar() + const theme = useTheme() + const query = useQuery() + const history = useHistory() + + const action = actionCreator() + const studies = useRecoilValue(studySummariesState) + const queried = useQueriedStudies(studies, query) + const selected = useMemo(() => { + return queried.length > 0 ? queried : studies.length > 0 ? [studies[0]] : [] + }, [studies, query]) + + const studyListWidth = 200 + const title = "Compare Studies" + + useEffect(() => { + action.updateStudySummaries() + }, []) + + const toolbar = ( + <> + + + + + + {title} + + + ) + + return ( + + + + + + + + {studies.length} Studies + + + + + {studies.map((study, i) => { + return ( + + { + if (e.shiftKey) { + let next: number[] + const selectedIds = selected.map((s) => s.study_id) + const alreadySelected = + selectedIds.findIndex( + (n) => n === study.study_id + ) >= 0 + if (alreadySelected) { + next = selectedIds.filter( + (n) => n !== study.study_id + ) + } else { + if ( + selected.length > 0 && + selected[0].directions.length !== + study.directions.length + ) { + enqueueSnackbar( + "You can only compare studies that has the same number of objectives.", + { + variant: "info", + } + ) + next = selectedIds + } else if ( + selected.length > 0 && + !isEqualDirections( + selected[0].directions, + study.directions + ) + ) { + enqueueSnackbar( + "You can only compare studies that has the same directions.", + { + variant: "info", + } + ) + next = selectedIds + } else { + next = [...selectedIds, study.study_id] + } + } + history.push(getStudyListLink(next)) + } else { + history.push(getStudyListLink([study.study_id])) + } + }} + selected={ + selected.findIndex( + (s) => s.study_id === study.study_id + ) !== -1 + } + sx={{ + display: "flex", + flexDirection: "column", + alignItems: "flex-start", + }} + > + + + + {`# ${study.study_id}`} + + + + + + ) + })} + + + + + + + + + + + + ) +} + +const StudiesGraph: FC<{ studies: StudySummary[] }> = ({ studies }) => { + const theme = useTheme() + const action = actionCreator() + const studyDetails = useRecoilValue(studyDetailsState) + const [logScale, setLogScale] = useState(false) + const [includePruned, setIncludePruned] = useState(true) + + const handleLogScaleChange = (e: ChangeEvent) => { + setLogScale(!logScale) + } + + const handleIncludePrunedChange = (e: ChangeEvent) => { + setIncludePruned(!includePruned) + } + + useEffect(() => { + studies.forEach((study) => { + action.updateStudyDetail(study.study_id) + }) + }, [studies]) + + const showStudyDetails = studies.map((study) => studyDetails[study.study_id]) + + return ( + + + + } + label="Log y scale" + /> + + } + label="Include PRUNED trials" + /> + + {showStudyDetails !== null && + showStudyDetails.length > 0 && + showStudyDetails.every((s) => s) ? ( + + + + + + ) : null} + {showStudyDetails !== null && + showStudyDetails.length > 0 && + showStudyDetails.every((s) => s) ? ( + + + + + + ) : null} + + ) +} diff --git a/optuna_dashboard/ts/components/GraphEdf.tsx b/optuna_dashboard/ts/components/GraphEdf.tsx index 0d8a4e45..ee1418fd 100644 --- a/optuna_dashboard/ts/components/GraphEdf.tsx +++ b/optuna_dashboard/ts/components/GraphEdf.tsx @@ -12,11 +12,21 @@ import { Box, } from "@mui/material" import { plotlyDarkTemplate } from "./PlotlyDarkMode" -import { Target, useFilteredTrials, useObjectiveTargets } from "../trialFilter" +import { + Target, + useFilteredTrials, + useFilteredTrialsFromStudies, + useObjectiveTargets, +} from "../trialFilter" const plotDomId = "graph-edf" const getPlotDomId = (objectiveId: number) => `graph-edf-${objectiveId}` +interface EdfPlotInfo { + study_name: string + trials: Trial[] +} + export const GraphEdfBeta: FC<{ study: StudyDetail | null objectiveId: number @@ -101,6 +111,69 @@ export const GraphEdf: FC<{ ) } +export const GraphEdfMultiStudies: FC<{ + studies: StudyDetail[] +}> = ({ studies }) => { + const theme = useTheme() + const [targets, selected, setTarget] = useObjectiveTargets( + studies.length !== 0 ? studies[0] : null + ) + + const trials = useFilteredTrialsFromStudies(studies, [selected], false, false) + const edfPlotInfos = studies.map((study, index) => { + const e: EdfPlotInfo = { + study_name: study?.name, + trials: trials[index], + } + return e + }) + + const handleObjectiveChange = (event: SelectChangeEvent) => { + setTarget(event.target.value) + } + + useEffect(() => { + plotEdfMultiStudies(edfPlotInfos, selected, plotDomId, theme.palette.mode) + }, [studies, selected, theme.palette.mode]) + + return ( + + + + EDF + + {studies.length > 0 && studies[0].directions.length !== 1 ? ( + + Objective: + + + ) : null} + + + + + + ) +} + const plotEdf = ( trials: Trial[], target: Target, @@ -159,3 +232,62 @@ const plotEdf = ( ] plotly.react(domId, plotData, layout) } + +const plotEdfMultiStudies = ( + edfPlotInfos: EdfPlotInfo[], + target: Target, + domId: string, + mode: string +) => { + if (document.getElementById(domId) === null) { + return + } + if (edfPlotInfos.length === 0) { + plotly.react(domId, [], { + template: mode === "dark" ? plotlyDarkTemplate : {}, + }) + return + } + + const target_name = "Objective Value" + const layout: Partial = { + xaxis: { + title: target_name, + }, + yaxis: { + title: "Cumulative Probability", + }, + margin: { + l: 50, + t: 0, + r: 50, + b: 50, + }, + template: mode === "dark" ? plotlyDarkTemplate : {}, + } + + const plotData: Partial[] = edfPlotInfos.map((h) => { + const values = h.trials.map((t) => target.getTargetValue(t) as number) + const numValues = values.length + const minX = Math.min(...values) + const maxX = Math.max(...values) + const numStep = 100 + const _step = (maxX - minX) / (numStep - 1) + + const xValues = [] + const yValues = [] + for (let i = 0; i < numStep; i++) { + const boundary_right = minX + _step * i + xValues.push(boundary_right) + yValues.push(values.filter((v) => v <= boundary_right).length / numValues) + } + + return { + type: "scatter", + name: `${h.study_name}`, + x: xValues, + y: yValues, + } + }) + plotly.react(domId, plotData, layout) +} diff --git a/optuna_dashboard/ts/components/GraphHistory.tsx b/optuna_dashboard/ts/components/GraphHistory.tsx index 512bbbe6..ae219850 100644 --- a/optuna_dashboard/ts/components/GraphHistory.tsx +++ b/optuna_dashboard/ts/components/GraphHistory.tsx @@ -18,12 +18,20 @@ import { import { plotlyDarkTemplate } from "./PlotlyDarkMode" import { useFilteredTrials, + useFilteredTrialsFromStudies, Target, useObjectiveAndUserAttrTargets, } from "../trialFilter" const plotDomId = "graph-history" +interface HistoryPlotInfo { + study_name: string + trials: Trial[] + directions: StudyDirection[] + objective_names?: string[] +} + export const GraphHistory: FC<{ study: StudyDetail | null betaLogScale?: boolean @@ -203,6 +211,185 @@ export const GraphHistory: FC<{ ) } +export const GraphHistoryMultiStudies: FC<{ + studies: StudyDetail[] + betaLogScale?: boolean + betaIncludePruned?: boolean +}> = ({ studies, betaLogScale, betaIncludePruned }) => { + const theme = useTheme() + const [xAxis, setXAxis] = useState< + "number" | "datetime_start" | "datetime_complete" + >("number") + const [logScale, setLogScale] = useState(false) + const [filterCompleteTrial, setFilterCompleteTrial] = useState(false) + const [filterPrunedTrial, setFilterPrunedTrial] = useState(false) + + // TODO(umezawa): Prepare targets with all studies. + const [targets, selected, setTarget] = useObjectiveAndUserAttrTargets( + studies.length !== 0 ? studies[0] : null + ) + + const trials = useFilteredTrialsFromStudies( + studies, + [selected], + filterCompleteTrial, + betaIncludePruned === undefined ? filterPrunedTrial : !betaIncludePruned + ) + const historyPlotInfos = studies.map((study, index) => { + const h: HistoryPlotInfo = { + study_name: study?.name, + trials: trials[index], + directions: study?.directions, + objective_names: study?.objective_names, + } + return h + }) + + useEffect(() => { + plotHistoryMultiStudies( + historyPlotInfos, + selected, + xAxis, + betaLogScale === undefined ? logScale : betaLogScale, + theme.palette.mode + ) + }, [studies, selected, logScale, betaLogScale, xAxis, theme.palette.mode]) + + const handleObjectiveChange = (event: SelectChangeEvent) => { + setTarget(event.target.value) + } + + const handleXAxisChange = (e: ChangeEvent) => { + if (e.target.value === "number") { + setXAxis("number") + } else if (e.target.value === "datetime_start") { + setXAxis("datetime_start") + } else if (e.target.value === "datetime_complete") { + setXAxis("datetime_complete") + } + } + + const handleLogScaleChange = (e: ChangeEvent) => { + setLogScale(!logScale) + } + + const handleFilterCompleteChange = (e: ChangeEvent) => { + setFilterCompleteTrial(!filterCompleteTrial) + } + + const handleFilterPrunedChange = (e: ChangeEvent) => { + setFilterPrunedTrial(!filterPrunedTrial) + } + + return ( + + + + History + + {studies[0] !== null && targets.length >= 2 ? ( + + y Axis + + + ) : null} + {betaLogScale === undefined ? ( + + Log y scale: + + + ) : null} + {betaIncludePruned === undefined ? ( + + Filter state: + + } + label="Complete" + /> + + } + label="Pruned" + /> + + ) : null} + + X-axis: + + } + label="Number" + /> + } + label="Datetime start" + /> + } + label="Datetime complete" + /> + + + + +
+ + + ) +} + const plotHistory = ( trials: Trial[], directions: StudyDirection[], @@ -307,3 +494,118 @@ const plotHistory = ( } plotly.react(plotDomId, plotData, layout) } + +const plotHistoryMultiStudies = ( + historyPlotInfos: HistoryPlotInfo[], + target: Target, + xAxis: "number" | "datetime_start" | "datetime_complete", + logScale: boolean, + mode: string +) => { + if (document.getElementById(plotDomId) === null) { + return + } + if (historyPlotInfos.length === 0) { + plotly.react(plotDomId, [], { + template: mode === "dark" ? plotlyDarkTemplate : {}, + }) + return + } + + const layout: Partial = { + margin: { + l: 50, + t: 0, + r: 50, + b: 0, + }, + yaxis: { + title: target.toLabel(historyPlotInfos[0].objective_names), + type: logScale ? "log" : "linear", + }, + xaxis: { + title: xAxis === "number" ? "Trial" : "Time", + type: xAxis === "number" ? "linear" : "date", + }, + showlegend: true, + template: mode === "dark" ? plotlyDarkTemplate : {}, + } + + const getAxisX = (trial: Trial): number | Date => { + return xAxis === "number" + ? trial.number + : xAxis === "datetime_start" + ? trial.datetime_start! + : trial.datetime_complete! + } + + const plotData: Partial[] = [] + historyPlotInfos.forEach((h) => { + const x = h.trials.map(getAxisX) + const y = h.trials.map( + (t: Trial): number => target.getTargetValue(t) as number + ) + plotData.push({ + x: x, + y: y, + name: `${target.toLabel(h.objective_names)} of ${h.study_name}`, + mode: "markers", + type: "scatter", + }) + + const objectiveId = target.getObjectiveId() + if (objectiveId !== null) { + const xForLinePlot: (number | Date)[] = [] + const yForLinePlot: number[] = [] + let currentBest: number | null = null + for (let i = 0; i < h.trials.length; i++) { + const t = h.trials[i] + const value = target.getTargetValue(t) as number + if (value === null) { + continue + } else if (currentBest === null) { + currentBest = value + xForLinePlot.push(getAxisX(t)) + yForLinePlot.push(value) + } else if ( + h.directions[objectiveId] === "maximize" && + value > currentBest + ) { + const p = h.trials[i - 1] + if (!xForLinePlot.includes(getAxisX(p))) { + xForLinePlot.push(getAxisX(p)) + yForLinePlot.push(currentBest) + } + currentBest = value + xForLinePlot.push(getAxisX(t)) + yForLinePlot.push(value) + } else if ( + h.directions[objectiveId] === "minimize" && + value < currentBest + ) { + const p = h.trials[i - 1] + if (!xForLinePlot.includes(getAxisX(p))) { + xForLinePlot.push(getAxisX(p)) + yForLinePlot.push(currentBest) + } + currentBest = value + xForLinePlot.push(getAxisX(t)) + yForLinePlot.push(value) + } + } + if (h.trials.length !== 0) { + xForLinePlot.push(getAxisX(h.trials[h.trials.length - 1])) + yForLinePlot.push(yForLinePlot[yForLinePlot.length - 1]) + } + plotData.push({ + x: xForLinePlot, + y: yForLinePlot, + name: `Best Value of ${h.study_name}`, + mode: "lines", + type: "scatter", + }) + } + }) + + plotly.react(plotDomId, plotData, layout) +} diff --git a/optuna_dashboard/ts/components/StudyListBeta.tsx b/optuna_dashboard/ts/components/StudyListBeta.tsx index c18ce8d9..9ed503d4 100644 --- a/optuna_dashboard/ts/components/StudyListBeta.tsx +++ b/optuna_dashboard/ts/components/StudyListBeta.tsx @@ -22,6 +22,7 @@ import { Delete, Refresh, Search } from "@mui/icons-material" import SortIcon from "@mui/icons-material/Sort" import HomeIcon from "@mui/icons-material/Home" import AddBoxIcon from "@mui/icons-material/AddBox" +import CompareIcon from "@mui/icons-material/Compare" import DriveFileRenameOutlineIcon from "@mui/icons-material/DriveFileRenameOutline" import { actionCreator } from "../action" @@ -180,6 +181,15 @@ export const StudyListBeta: FC<{ > Create + diff --git a/optuna_dashboard/ts/trialFilter.ts b/optuna_dashboard/ts/trialFilter.ts index e3424956..2cb95793 100644 --- a/optuna_dashboard/ts/trialFilter.ts +++ b/optuna_dashboard/ts/trialFilter.ts @@ -92,6 +92,29 @@ export class Target { } } +const filterTrials = ( + study: StudyDetail | null, + targets: Target[], + filterComplete: boolean, + filterPruned: boolean +): Trial[] => { + if (study === null) { + return [] + } + return study.trials.filter((t) => { + if (t.state !== "Complete" && t.state !== "Pruned") { + return false + } + if (t.state === "Complete" && filterComplete) { + return false + } + if (t.state === "Pruned" && filterPruned) { + return false + } + return targets.every((target) => target.getTargetValue(t) !== null) + }) +} + export const useFilteredTrials = ( study: StudyDetail | null, targets: Target[], @@ -99,23 +122,21 @@ export const useFilteredTrials = ( filterPruned: boolean ): Trial[] => useMemo(() => { - if (study === null) { - return [] - } - return study.trials.filter((t) => { - if (t.state !== "Complete" && t.state !== "Pruned") { - return false - } - if (t.state === "Complete" && filterComplete) { - return false - } - if (t.state === "Pruned" && filterPruned) { - return false - } - return targets.every((target) => target.getTargetValue(t) !== null) - }) + return filterTrials(study, targets, filterComplete, filterPruned) }, [study?.trials, targets, filterComplete, filterPruned]) +export const useFilteredTrialsFromStudies = ( + studies: StudyDetail[], + targets: Target[], + filterComplete: boolean, + filterPruned: boolean +): Trial[][] => + useMemo(() => { + return studies.map((s) => + filterTrials(s, targets, filterComplete, filterPruned) + ) + }, [studies, targets, filterComplete, filterPruned]) + export const useObjectiveTargets = ( study: StudyDetail | null ): [Target[], Target, (ident: string) => void] => {