From 516f06a19c49fe674336b4f74575eec64e4b74dc Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 31 Aug 2023 14:08:05 +0900 Subject: [PATCH 01/38] wip --- .../ts/components/PreferentialTrials.tsx | 138 +++- optuna_dashboard/ts/components/TrialList.tsx | 727 ++++++++++-------- optuna_dashboard/ts/state.ts | 11 + 3 files changed, 535 insertions(+), 341 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 5c141403..1129d2a0 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -7,17 +7,59 @@ import { CardContent, CardActions, CardActionArea, + CardMedia, + MenuItem, + Select, + FormControl, + FormLabel, + TextField, + Modal, } from "@mui/material" -import ClearIcon from "@mui/icons-material/Clear" -import IconButton from "@mui/material/IconButton" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import ReplayIcon from "@mui/icons-material/Replay" -import Modal from "@mui/material/Modal" -import { red } from "@mui/material/colors" - +import ClearIcon from "@mui/icons-material/Clear" +import IconButton from "@mui/material/IconButton" +import SettingsIcon from "@mui/icons-material/Settings" +import red from "@mui/material/colors/red" +import { useRecoilValue, useSetRecoilState } from "recoil" import { actionCreator } from "../action" -import { TrialListDetail } from "./TrialList" import { MarkdownRenderer } from "./Note" +import { + feedbackComponent, + FeedbackComponentType, + feedbackArtifactKey, +} from "../state" +import { + TrialArtifactActions, + TrialArtifactContent, + TrialListDetail, +} from "./TrialList" + +const FeedbackContent: FC<{ + trial: Trial + artifact?: Artifact +}> = ({ trial, artifact }) => { + const componentId = useRecoilValue(feedbackComponent) + + if (componentId === "note") { + return + } + if (componentId === "artifact") { + if (artifact === undefined) { + return null + } + return ( + + ) + } + + return null +} const PreferentialTrial: FC<{ trial?: Trial @@ -29,6 +71,9 @@ const PreferentialTrial: FC<{ const trialWidth = 500 const trialHeight = 300 const [detailShown, setDetailShown] = useState(false) + const componentId = useRecoilValue(feedbackComponent) + const artifactKey = useRecoilValue(feedbackArtifactKey) + const artifact = trial?.artifacts.find((a) => a.filename === artifactKey) if (trial == undefined) { return ( @@ -52,7 +97,19 @@ const PreferentialTrial: FC<{ }} > - Trial {trial.number} + + Trial {trial.number} + {componentId === "artifact" && artifact !== undefined + ? ` (${artifact.filename})` + : ""} + + {componentId === "artifact" && artifact !== undefined ? ( + + ) : null} - + = ({ numbers: studyDetail.best_trials.map((t) => t.number), last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1), }) + const [settingShown, setSettingShown] = useState(false) + const outputComponent = useRecoilValue(feedbackComponent) + const setOutputComponent = useSetRecoilState(feedbackComponent) + const outputartifactKey = useRecoilValue(feedbackArtifactKey) + const setOutputartifactKey = useSetRecoilState(feedbackArtifactKey) const new_trails = studyDetail.best_trials.filter( (t) => displayTrials.last_number < t.number && @@ -228,7 +290,23 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ } return ( - + + setSettingShown(true)} + > + + = ({ /> ))} + + + Settings + + + Output Component: + + + {outputComponent === "artifact" ? ( + + Output File: + { + setOutputartifactKey(e.target.value) + }} + /> + + ) : null} + + + ) } diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 4aceb619..25ec9fab 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -22,6 +22,7 @@ import { CardActionArea, Modal, } from "@mui/material" +import { SxProps } from "@mui/system" import Chip from "@mui/material/Chip" import Divider from "@mui/material/Divider" import List from "@mui/material/List" @@ -319,16 +320,397 @@ export const TrialListDetail: FC<{ value !== null ? renderInfo(key, value) : null )} - {artifactEnabled && } + {artifactEnabled && } ) } -const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { +export const TrialArtifactContent: FC<{ + trial: Trial + artifact: Artifact + width: string + height: string +}> = ({ trial, artifact, width, height }) => { + if (artifact.mimetype.startsWith("image")) { + return ( + + ) + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + + + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + + + ) + } else { + return ( + + + + ) + } +} + +export const TrialArtifactActions: FC<{ + trial: Trial + artifact: Artifact + sx: SxProps +}> = ({ trial, artifact, sx }) => { + const [open3dModelViewer, setOpen3dModelViewer] = useState(false) + + if (artifact.mimetype.startsWith("image")) { + return null + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + <> + { + setOpen3dModelViewer(true) + }} + > + + + { + setOpen3dModelViewer(false) + }} + > + + + + + + ) + } + return null +} + +const TrialArtifact: FC<{ + trial: Trial + artifact: Artifact + width: string + height: string +}> = ({ trial, artifact, width, height }) => { + const [openDeleteArtifactDialog, _] = useDeleteArtifactDialog() + const theme = useTheme() + if (artifact.mimetype.startsWith("image")) { + return ( + + + + + {artifact.filename} + + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + + + + + + ) + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + + + + + {artifact.filename} + + + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + + + + + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + + + + {artifact.filename} + + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + + + + + + ) + } else { + return ( + + + + + {artifact.filename} + + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + + + + + + ) + } +} + +const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { const theme = useTheme() const action = actionCreator() - const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = - useDeleteArtifactDialog() + const [_, renderDeleteArtifactDialog] = useDeleteArtifactDialog() const [dragOver, setDragOver] = useState(false) const [open3dModelViewer, setOpen3dModelViewer] = useState<{ [key: string]: boolean @@ -382,334 +764,15 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { Artifacts - {trial.artifacts.map((a) => { - if (a.mimetype.startsWith("image")) { - return ( - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if ( - a.filename.endsWith(".stl") || - a.filename.endsWith(".3dm") - ) { - return ( - - - - - - - {a.filename} - - { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = true - return obj - }) - }} - > - - - { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = false - return obj - }) - }} - > - - - - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if (a.mimetype.startsWith("audio")) { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } - })} + {trial.artifacts.map((a) => ( + + ))} {trial.state === "Running" || trial.state === "Waiting" ? ( ({ default: false, }) +export type FeedbackComponentType = "note" | "artifact" +export const feedbackComponent = atom({ + key: "feedbackComponent", + default: "note", +}) + +export const feedbackArtifactKey = atom({ + key: "feedbackArtifactKey", + default: "", +}) + export const useStudyDetailValue = (studyId: number): StudyDetail | null => { const studyDetails = useRecoilValue(studyDetailsState) return studyDetails[studyId] || null From 181319e89cfe0deef3b7deb02723d88a08772c21 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 31 Aug 2023 15:24:08 +0900 Subject: [PATCH 02/38] add setting to frontend --- .../ts/components/PreferentialTrials.tsx | 136 +++++++++++------- optuna_dashboard/ts/components/TrialList.tsx | 7 +- 2 files changed, 89 insertions(+), 54 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 1129d2a0..06bdad9d 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -7,7 +7,6 @@ import { CardContent, CardActions, CardActionArea, - CardMedia, MenuItem, Select, FormControl, @@ -61,6 +60,43 @@ const FeedbackContent: FC<{ return null } +const ModalPage: FC<{ + children: React.ReactNode + displayFlag: boolean + setDisplayFlag: (flag: boolean) => void +}> = ({ children, displayFlag, setDisplayFlag }) => { + const theme = useTheme() + return ( + setDisplayFlag(false)}> + + + {children} + + + + ) +} + const PreferentialTrial: FC<{ trial?: Trial studyDetail: StudyDetail @@ -73,7 +109,8 @@ const PreferentialTrial: FC<{ const [detailShown, setDetailShown] = useState(false) const componentId = useRecoilValue(feedbackComponent) const artifactKey = useRecoilValue(feedbackArtifactKey) - const artifact = trial?.artifacts.find((a) => a.filename === artifactKey) + const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value + const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) if (trial == undefined) { return ( @@ -97,12 +134,17 @@ const PreferentialTrial: FC<{ }} > - - Trial {trial.number} - {componentId === "artifact" && artifact !== undefined - ? ` (${artifact.filename})` - : ""} - + Trial {trial.number} + {componentId === "artifact" && artifact !== undefined ? ( + + {`(${artifact.filename})`} + + ) : null} {componentId === "artifact" && artifact !== undefined ? ( - setDetailShown(false)}> - - - true} - directions={[]} - objectiveNames={[]} - /> - - - + + true} + directions={[]} + objectiveNames={[]} + /> + ) } @@ -249,8 +267,8 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ const [settingShown, setSettingShown] = useState(false) const outputComponent = useRecoilValue(feedbackComponent) const setOutputComponent = useSetRecoilState(feedbackComponent) - const outputartifactKey = useRecoilValue(feedbackArtifactKey) - const setOutputartifactKey = useSetRecoilState(feedbackArtifactKey) + const outputArtifactKey = useRecoilValue(feedbackArtifactKey) + const setOutputArtifactKey = useSetRecoilState(feedbackArtifactKey) const new_trails = studyDetail.best_trials.filter( (t) => displayTrials.last_number < t.number && @@ -357,14 +375,34 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {outputComponent === "artifact" ? ( - - Output File: - { - setOutputartifactKey(e.target.value) + + + User Attribute Key Corresponding to Output Artifact Id: + + ) : null} diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 25ec9fab..f73a97e1 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -459,7 +459,7 @@ const TrialArtifact: FC<{ width: string height: string }> = ({ trial, artifact, width, height }) => { - const [openDeleteArtifactDialog, _] = useDeleteArtifactDialog() + const [openDeleteArtifactDialog] = useDeleteArtifactDialog() const theme = useTheme() if (artifact.mimetype.startsWith("image")) { return ( @@ -710,11 +710,8 @@ const TrialArtifact: FC<{ const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { const theme = useTheme() const action = actionCreator() - const [_, renderDeleteArtifactDialog] = useDeleteArtifactDialog() + const [, renderDeleteArtifactDialog] = useDeleteArtifactDialog() const [dragOver, setDragOver] = useState(false) - const [open3dModelViewer, setOpen3dModelViewer] = useState<{ - [key: string]: boolean - }>({}) const width = "200px" const height = "150px" From 6928e0c304e61648b9cba28f1b48ef47c63da99e Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 4 Sep 2023 15:32:35 +0900 Subject: [PATCH 03/38] add python api --- optuna_dashboard/_app.py | 23 ++ optuna_dashboard/_preference_setting.py | 60 +++++ optuna_dashboard/_serializer.py | 6 + optuna_dashboard/ts/action.ts | 21 ++ optuna_dashboard/ts/apiClient.ts | 20 ++ .../ts/components/PreferentialTrials.tsx | 212 ++++++++++-------- optuna_dashboard/ts/state.ts | 11 - optuna_dashboard/ts/types/index.d.ts | 3 + 8 files changed, 257 insertions(+), 99 deletions(-) create mode 100644 optuna_dashboard/_preference_setting.py diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 1c439408..70acc13f 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -27,6 +27,7 @@ from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preference_setting import _register_output_component from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -284,6 +285,28 @@ def create_app( response.status = 204 return {} + @app.post("/api/studies//component") + @json_api_view + def post_component(study_id: int) -> dict[str, Any]: + try: + component_type = request.json.get("component_type", "") + artifact_key = request.json.get("artifact_key", None) + except ValueError: + response.status = 400 + return {"reason": "invalid request."} + if component_type not in ["Note", "Artifact"]: + response.status = 400 + return {"reason": "component_type must be either 'Note' or 'Artifact'."} + + _register_output_component( + study_id=study_id, + storage=storage, + component_type=component_type, + artifact_key=artifact_key, + ) + response.status = 204 + return {} + @app.post("/api/trials//tell") @json_api_view def tell_trial(trial_id: int) -> dict[str, Any]: diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py new file mode 100644 index 00000000..93b1a34c --- /dev/null +++ b/optuna_dashboard/_preference_setting.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from optuna.storages import BaseStorage + +from .preferential._study import PreferentialStudy + + +if TYPE_CHECKING: + from typing import Literal + + OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] + +_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE = "preference:component_type" +_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY = "preference:component_artifact_key" + + +def _register_output_component( + study_id: int, + storage: BaseStorage, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str | None = None, +) -> None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, + value=component_type, + ) + if artifact_key is not None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, + value=artifact_key, + ) + + +def register_output_component( + study: PreferentialStudy, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str = "", +) -> None: + """Register output component to the study. + + Args: + study: + The study to register the output component. + component_type: + The type of the output component. + artifact_key: + When the component_type is "Artifact", + this argument is used as the attribute key of the artifact. + Each trial displays the artifact whose id is the value of the attribute. + """ + _register_output_component( + study_id=study._study._study_id, + storage=study._study._storage, + component_type=component_type, + artifact_key=artifact_key, + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 0fa04124..b2494068 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -14,6 +14,8 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -155,6 +157,10 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: + serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] + if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: + serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] return serialized diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index 017751fa..f0c82e2b 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -16,6 +16,7 @@ import { deleteArtifactAPI, reportPreferenceAPI, skipPreferentialTrialAPI, + reportFeedbackComponentAPI, } from "./apiClient" import { graphVisibilityState, @@ -609,6 +610,25 @@ export const actionCreator = () => { }) } + const updateFeedbackComponent = ( + studyId: number, + compoennt_type: FeedbackComponentType, + artifact_key?: string + ) => { + reportFeedbackComponentAPI(studyId, compoennt_type, artifact_key).catch( + (err) => { + const reason = err.response?.data.reason + enqueueSnackbar( + `Failed to report feedback component. Reason: ${reason}`, + { + variant: "error", + } + ) + console.log(err) + } + ) + } + return { updateAPIMeta, updateStudyDetail, @@ -630,6 +650,7 @@ export const actionCreator = () => { saveTrialUserAttrs, updatePreference, skipPreferentialTrial, + updateFeedbackComponent, } } diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 25ca3541..caa57716 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -70,6 +70,8 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + feedback_component_type?: string + feedback_artifact_key?: string } export const getStudyDetailAPI = ( @@ -105,6 +107,9 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, + feedback_component_type: res.data + .feedback_component_type as FeedbackComponentType, + feedback_artifact_key: res.data.feedback_artifact_key, } }) } @@ -337,3 +342,18 @@ export const skipPreferentialTrialAPI = ( return }) } + +export const reportFeedbackComponentAPI = ( + studyId: number, + component_type: FeedbackComponentType, + artifact_key?: string +): Promise => { + return axiosInstance + .post(`/api/studies/${studyId}/component`, { + component_type: component_type, + artifact_key: artifact_key, + }) + .then(() => { + return + }) +} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 06bdad9d..a5d1e4af 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -1,4 +1,4 @@ -import React, { FC, useState } from "react" +import React, { FC, useEffect, useState } from "react" import { Typography, Box, @@ -11,7 +11,6 @@ import { Select, FormControl, FormLabel, - TextField, Modal, } from "@mui/material" import OpenInFullIcon from "@mui/icons-material/OpenInFull" @@ -20,14 +19,8 @@ import ClearIcon from "@mui/icons-material/Clear" import IconButton from "@mui/material/IconButton" import SettingsIcon from "@mui/icons-material/Settings" import red from "@mui/material/colors/red" -import { useRecoilValue, useSetRecoilState } from "recoil" import { actionCreator } from "../action" import { MarkdownRenderer } from "./Note" -import { - feedbackComponent, - FeedbackComponentType, - feedbackArtifactKey, -} from "../state" import { TrialArtifactActions, TrialArtifactContent, @@ -37,13 +30,12 @@ import { const FeedbackContent: FC<{ trial: Trial artifact?: Artifact -}> = ({ trial, artifact }) => { - const componentId = useRecoilValue(feedbackComponent) - - if (componentId === "note") { + componentId: FeedbackComponentType +}> = ({ trial, artifact, componentId }) => { + if (componentId === "Note") { return } - if (componentId === "artifact") { + if (componentId === "Artifact") { if (artifact === undefined) { return null } @@ -63,11 +55,11 @@ const FeedbackContent: FC<{ const ModalPage: FC<{ children: React.ReactNode displayFlag: boolean - setDisplayFlag: (flag: boolean) => void -}> = ({ children, displayFlag, setDisplayFlag }) => { + onClose: () => void +}> = ({ children, displayFlag, onClose }) => { const theme = useTheme() return ( - setDisplayFlag(false)}> + a.key === artifactKey)?.value const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) @@ -135,7 +127,7 @@ const PreferentialTrial: FC<{ > Trial {trial.number} - {componentId === "artifact" && artifact !== undefined ? ( + {componentId === "Artifact" && artifact !== undefined ? ( ) : null} - {componentId === "artifact" && artifact !== undefined ? ( + {componentId === "Artifact" && artifact !== undefined ? ( - + - + { + setDetailShown(false) + }} + > true} @@ -248,6 +249,102 @@ const PreferentialTrial: FC<{ ) } +const SettingsPage: FC<{ + studyDetail: StudyDetail + settingShown: boolean + setSettingShown: (flag: boolean) => void +}> = ({ studyDetail, settingShown, setSettingShown }) => { + const theme = useTheme() + const actions = actionCreator() + const [outputComponent, setOutputComponent] = useState( + studyDetail?.feedback_component_type ?? "Note" + ) + const [outputArtifactKey, setOutputArtifactKey] = useState( + studyDetail?.feedback_artifact_key ?? "" + ) + useEffect(() => { + if (studyDetail.feedback_component_type !== undefined) { + setOutputComponent(studyDetail.feedback_component_type) + } + if (studyDetail.feedback_artifact_key !== undefined) { + setOutputArtifactKey(studyDetail.feedback_artifact_key) + } + }, [studyDetail.feedback_component_type, studyDetail.feedback_artifact_key]) + const onClose = () => { + setSettingShown(false) + actions.updateFeedbackComponent( + studyDetail.id, + outputComponent, + outputArtifactKey + ) + } + + return ( + + + Settings + + + + Output Component: + + + {outputComponent === "Artifact" ? ( + + + User Attribute Key Corresponding to Output Artifact Id: + + + + ) : null} + + + ) +} + type DisplayTrials = { numbers: number[] last_number: number @@ -265,10 +362,6 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1), }) const [settingShown, setSettingShown] = useState(false) - const outputComponent = useRecoilValue(feedbackComponent) - const setOutputComponent = useSetRecoilState(feedbackComponent) - const outputArtifactKey = useRecoilValue(feedbackArtifactKey) - const setOutputArtifactKey = useSetRecoilState(feedbackArtifactKey) const new_trails = studyDetail.best_trials.filter( (t) => displayTrials.last_number < t.number && @@ -346,68 +439,11 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ /> ))} - - - Settings - - - Output Component: - - - {outputComponent === "artifact" ? ( - - - User Attribute Key Corresponding to Output Artifact Id: - - - - ) : null} - - - + ) } diff --git a/optuna_dashboard/ts/state.ts b/optuna_dashboard/ts/state.ts index 7464adb3..44301bd1 100644 --- a/optuna_dashboard/ts/state.ts +++ b/optuna_dashboard/ts/state.ts @@ -56,17 +56,6 @@ export const artifactIsAvailable = atom({ default: false, }) -export type FeedbackComponentType = "note" | "artifact" -export const feedbackComponent = atom({ - key: "feedbackComponent", - default: "note", -}) - -export const feedbackArtifactKey = atom({ - key: "feedbackArtifactKey", - default: "", -}) - export const useStudyDetailValue = (studyId: number): StudyDetail | null => { const studyDetails = useRecoilValue(studyDetailsState) return studyDetails[studyId] || null diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 1720cc6b..241ea310 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -12,6 +12,7 @@ type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type TrialStateFinished = "Complete" | "Fail" | "Pruned" type StudyDirection = "maximize" | "minimize" | "not_set" +type FeedbackComponentType = "Note" | "Artifact" type FloatDistribution = { type: "FloatDistribution" @@ -197,6 +198,8 @@ type StudyDetail = { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + feedback_component_type?: FeedbackComponentType + feedback_artifact_key?: string } type StudyDetails = { From da38286b69d9a16784b5730e3b65416f49bb7722 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 4 Sep 2023 18:59:17 +0900 Subject: [PATCH 04/38] add tests --- optuna_dashboard/ts/components/TrialList.tsx | 284 ++++--------------- python_tests/test_api.py | 63 ++++ python_tests/test_preference_setting.py | 20 ++ 3 files changed, 138 insertions(+), 229 deletions(-) create mode 100644 python_tests/test_preference_setting.py diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index f73a97e1..3a1fcee6 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -370,6 +370,7 @@ export const TrialArtifactContent: FC<{ display: "flex", justifyContent: "center", alignItems: "center", + height: height, }} > + - - {artifact.filename} - - { - openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) - }} - > - - - - - - - - ) - } + + + + + ) } const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { diff --git a/python_tests/test_api.py b/python_tests/test_api.py index ae50e29a..6828b83d 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -9,6 +9,7 @@ from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study from optuna_dashboard.preferential import create_study +from optuna_dashboard._preference_setting import register_output_component from .wsgi_client import send_request @@ -151,6 +152,68 @@ class APITestCase(TestCase): assert better.number == 2 assert worse.number == 1 + def test_change_component(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage) + register_output_component(study, "Note") + for _ in range(3): + trial = study.ask() + study.mark_comparison_ready(trial) + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Artifact", "artifact_key": "image"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Artifact" + assert study_detail["feedback_artifact_key"] == "image" + + def test_change_component_type_only(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage) + register_output_component(study, "Artifact", "audio") + for _ in range(3): + trial = study.ask() + study.mark_comparison_ready(trial) + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Note"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Note" + assert study_detail["feedback_artifact_key"] == "audio" + def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage) diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py new file mode 100644 index 00000000..05727a04 --- /dev/null +++ b/python_tests/test_preference_setting.py @@ -0,0 +1,20 @@ +from __future__ import annotations +from unittest import TestCase + +import optuna + +from optuna_dashboard._preference_setting import ( + register_output_component, + _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, + _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, +) +from optuna_dashboard.preferential._study import PreferentialStudy + + +class FeedbackSettingTestCase(TestCase): + def test_widget_to_dict_from_dict(self) -> None: + study = PreferentialStudy(optuna.create_study()) + register_output_component(study, "Artifact", "image_key") + system_attrs = study._study.system_attrs + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, "") == "Artifact" + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, "") == "image_key" From 5cbab3fa6791a3a6f8e013b760dfbbe3d72138af Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 4 Sep 2023 19:01:06 +0900 Subject: [PATCH 05/38] fix by lint --- python_tests/test_api.py | 2 +- python_tests/test_preference_setting.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 6828b83d..51017ac8 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,8 +8,8 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard.preferential import create_study from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard.preferential import create_study from .wsgi_client import send_request diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index 05727a04..033a9092 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -1,13 +1,11 @@ from __future__ import annotations + from unittest import TestCase import optuna - -from optuna_dashboard._preference_setting import ( - register_output_component, - _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, - _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, -) +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from optuna_dashboard._preference_setting import register_output_component from optuna_dashboard.preferential._study import PreferentialStudy From bfb9afc898183cffb14b8bf3022bfce429fb9de3 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 5 Sep 2023 14:58:59 +0900 Subject: [PATCH 06/38] fixed Feedback screen --- .../ts/components/PreferentialTrials.tsx | 151 ++++++++++-------- optuna_dashboard/ts/components/TrialList.tsx | 45 +++--- 2 files changed, 107 insertions(+), 89 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index a5d1e4af..ec597a09 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -6,7 +6,7 @@ import { Card, CardContent, CardActions, - CardActionArea, + Button, MenuItem, Select, FormControl, @@ -31,7 +31,9 @@ const FeedbackContent: FC<{ trial: Trial artifact?: Artifact componentId: FeedbackComponentType -}> = ({ trial, artifact, componentId }) => { + width: string + minHeight: string +}> = ({ trial, artifact, componentId, width, minHeight }) => { if (componentId === "Note") { return } @@ -43,8 +45,8 @@ const FeedbackContent: FC<{ ) } @@ -96,9 +98,10 @@ const PreferentialTrial: FC<{ }> = ({ trial, studyDetail, hideTrial }) => { const theme = useTheme() const action = actionCreator() - const trialWidth = 500 + const trialWidth = 400 const trialHeight = 300 const [detailShown, setDetailShown] = useState(false) + const [buttonHover, setButtonHover] = useState(false) const componentId = studyDetail.feedback_component_type ?? "Note" const artifactKey = studyDetail.feedback_artifact_key const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value @@ -115,6 +118,13 @@ const PreferentialTrial: FC<{ /> ) } + const onFeedback = () => { + hideTrial() + const best_trials = studyDetail.best_trials + .map((t) => t.number) + .filter((t) => t !== trial.number) + action.updatePreference(trial.study_id, best_trials, [trial.number]) + } return ( - - { - hideTrial() - const best_trials = studyDetail.best_trials - .map((t) => t.number) - .filter((t) => t !== trial.number) - action.updatePreference(trial.study_id, best_trials, [trial.number]) - }} + { + if (e.shiftKey) onFeedback() + }} + sx={{ + position: "relative", + padding: theme.spacing(2), + overflow: "hidden", + minHeight: theme.spacing(20), + }} + > + + + + + + + { @@ -430,7 +443,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {displayTrials.numbers.map((t, index) => ( trial.number === t)} studyDetail={studyDetail} hideTrial={() => { diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 3a1fcee6..bb70c119 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -459,13 +459,17 @@ const TrialArtifact: FC<{ artifact: Artifact width: string height: string - buttons_width: number }> = ({ trial, artifact, width, height }) => { - const [openDeleteArtifactDialog] = useDeleteArtifactDialog() + const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = + useDeleteArtifactDialog() const theme = useTheme() - const is_3d_model = + const is3dModel = artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") - const actions_width = is_3d_model ? theme.spacing(12) : theme.spacing(8) + const canDelete = trial.state === "Running" || trial.state === "Waiting" + let actionsCount = 1 + if (canDelete) actionsCount += 1 + if (is3dModel) actionsCount += 1 + const actionsWidth = theme.spacing(actionsCount * 4) return ( @@ -495,29 +499,31 @@ const TrialArtifact: FC<{ p: theme.spacing(0.5, 0), flexGrow: 1, wordWrap: "break-word", - maxWidth: `calc(100% - ${actions_width})`, + maxWidth: `calc(100% - ${actionsWidth})`, }} > {artifact.filename} - {is_3d_model ? ( + {is3dModel ? ( ) : null} - { - openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) - }} - > - - + {canDelete ? ( + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + ) : null} + {renderDeleteArtifactDialog()} ) } @@ -536,7 +543,6 @@ const TrialArtifact: FC<{ const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { const theme = useTheme() const action = actionCreator() - const [, renderDeleteArtifactDialog] = useDeleteArtifactDialog() const [dragOver, setDragOver] = useState(false) const width = "200px" @@ -648,7 +654,6 @@ const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { ) : null} - {renderDeleteArtifactDialog()} ) } From 3f586020a444e701dd676032c53af767ca753715 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 6 Sep 2023 11:15:06 +0900 Subject: [PATCH 07/38] refactor --- optuna_dashboard/ts/components/TrialList.tsx | 563 ++++++++----------- 1 file changed, 227 insertions(+), 336 deletions(-) diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 4aceb619..bb70c119 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -22,6 +22,7 @@ import { CardActionArea, Modal, } from "@mui/material" +import { SxProps } from "@mui/system" import Chip from "@mui/material/Chip" import Divider from "@mui/material/Divider" import List from "@mui/material/List" @@ -319,20 +320,230 @@ export const TrialListDetail: FC<{ value !== null ? renderInfo(key, value) : null )} - {artifactEnabled && } + {artifactEnabled && } ) } -const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { - const theme = useTheme() - const action = actionCreator() +export const TrialArtifactContent: FC<{ + trial: Trial + artifact: Artifact + width: string + height: string +}> = ({ trial, artifact, width, height }) => { + if (artifact.mimetype.startsWith("image")) { + return ( + + ) + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + + + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + + + ) + } else { + return ( + + + + ) + } +} + +export const TrialArtifactActions: FC<{ + trial: Trial + artifact: Artifact + sx: SxProps +}> = ({ trial, artifact, sx }) => { + const [open3dModelViewer, setOpen3dModelViewer] = useState(false) + + if (artifact.mimetype.startsWith("image")) { + return null + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + <> + { + setOpen3dModelViewer(true) + }} + > + + + { + setOpen3dModelViewer(false) + }} + > + + + + + + ) + } + return null +} + +const TrialArtifact: FC<{ + trial: Trial + artifact: Artifact + width: string + height: string +}> = ({ trial, artifact, width, height }) => { const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = useDeleteArtifactDialog() + const theme = useTheme() + const is3dModel = + artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") + const canDelete = trial.state === "Running" || trial.state === "Waiting" + let actionsCount = 1 + if (canDelete) actionsCount += 1 + if (is3dModel) actionsCount += 1 + const actionsWidth = theme.spacing(actionsCount * 4) + + return ( + + + + + {artifact.filename} + + {is3dModel ? ( + + ) : null} + {canDelete ? ( + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + ) : null} + + + + + {renderDeleteArtifactDialog()} + + ) +} + +const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { + const theme = useTheme() + const action = actionCreator() const [dragOver, setDragOver] = useState(false) - const [open3dModelViewer, setOpen3dModelViewer] = useState<{ - [key: string]: boolean - }>({}) const width = "200px" const height = "150px" @@ -382,334 +593,15 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { Artifacts - {trial.artifacts.map((a) => { - if (a.mimetype.startsWith("image")) { - return ( - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if ( - a.filename.endsWith(".stl") || - a.filename.endsWith(".3dm") - ) { - return ( - - - - - - - {a.filename} - - { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = true - return obj - }) - }} - > - - - { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = false - return obj - }) - }} - > - - - - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if (a.mimetype.startsWith("audio")) { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } - })} + {trial.artifacts.map((a) => ( + + ))} {trial.state === "Running" || trial.state === "Waiting" ? ( = ({ trial }) => { ) : null} - {renderDeleteArtifactDialog()} ) } From 92b94b64888d9b6b805b169ec2cc9e7994ee65e4 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 14:40:57 +0900 Subject: [PATCH 08/38] merge and modify --- .github/workflows/python-coverage.yml | 2 +- docs/api.rst | 1 + .../preferential-optimization/generator.py | 3 - optuna_dashboard/__init__.py | 3 +- optuna_dashboard/_app.py | 40 +- optuna_dashboard/_custom_plot_data.py | 134 ++++ optuna_dashboard/_preferential_history.py | 80 +++ optuna_dashboard/_serializer.py | 35 ++ optuna_dashboard/preferential/_study.py | 94 +-- .../preferential/_system_attrs.py | 27 +- optuna_dashboard/preferential/samplers/gp.py | 579 ++++++++---------- optuna_dashboard/ts/action.ts | 6 +- optuna_dashboard/ts/apiClient.ts | 39 +- optuna_dashboard/ts/components/App.tsx | 9 + optuna_dashboard/ts/components/AppDrawer.tsx | 31 +- .../ts/components/ArtifactCardMedia.tsx | 41 ++ .../ts/components/PreferenceHistory.tsx | 218 +++++++ .../ts/components/PreferentialTrials.tsx | 473 +++++++------- .../ts/components/StudyDetail.tsx | 3 + .../ts/components/StudyHistory.tsx | 11 + .../ts/components/ThreejsArtifactViewer.tsx | 54 +- .../ts/components/TrialArtifactCards.tsx | 233 +++++++ optuna_dashboard/ts/components/TrialList.tsx | 360 +---------- .../ts/components/UserDefinedPlot.tsx | 21 + optuna_dashboard/ts/types/index.d.ts | 17 + pyproject.toml | 1 + python_tests/preferential/test_study.py | 6 +- .../preferential/test_system_attrs.py | 7 +- python_tests/test_api.py | 57 +- python_tests/test_custom_plot_data.py | 86 +++ python_tests/test_preferential_history.py | 61 ++ python_tests/test_serializers.py | 4 +- 32 files changed, 1755 insertions(+), 981 deletions(-) create mode 100644 optuna_dashboard/_custom_plot_data.py create mode 100644 optuna_dashboard/_preferential_history.py create mode 100644 optuna_dashboard/ts/components/ArtifactCardMedia.tsx create mode 100644 optuna_dashboard/ts/components/PreferenceHistory.tsx create mode 100644 optuna_dashboard/ts/components/TrialArtifactCards.tsx create mode 100644 optuna_dashboard/ts/components/UserDefinedPlot.tsx create mode 100644 python_tests/test_custom_plot_data.py create mode 100644 python_tests/test_preferential_history.py diff --git a/.github/workflows/python-coverage.yml b/.github/workflows/python-coverage.yml index 4bda6350..c96f506b 100644 --- a/.github/workflows/python-coverage.yml +++ b/.github/workflows/python-coverage.yml @@ -45,4 +45,4 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml - fail_ci_if_error: true + fail_ci_if_error: false diff --git a/docs/api.rst b/docs/api.rst index 09a9c14b..aadd2718 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -14,6 +14,7 @@ General APIs optuna_dashboard.wsgi optuna_dashboard.set_objective_names optuna_dashboard.save_note + optuna_dashboard.save_plotly_graph_object Human-in-the-loop ----------------- diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index e94f1d05..d8f7d3bd 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -64,9 +64,6 @@ def main() -> NoReturn: ) save_note(trial, note) - # 5. Mark comparison ready - study.mark_comparison_ready(trial) - if __name__ == "__main__": main() diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 3d363cf4..5bb3f301 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -1,5 +1,6 @@ from ._app import run_server # noqa from ._app import wsgi # noqa +from ._custom_plot_data import save_plotly_graph_object # noqa from ._form_widget import ChoiceWidget # noqa from ._form_widget import dict_to_form_widget # noqa from ._form_widget import ObjectiveChoiceWidget # noqa @@ -15,4 +16,4 @@ from ._note import get_note # noqa from ._note import save_note # noqa -__version__ = "0.12.0" +__version__ = "0.13.0b1" diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 70acc13f..fad4b74c 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -25,9 +25,12 @@ from . import _note as note from ._bottle_util import BottleViewReturn from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property +from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials from ._preference_setting import _register_output_component +from ._preferential_history import NewHistory +from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -41,7 +44,6 @@ from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._study import get_best_trials as get_best_preferential_trials -from .preferential._system_attrs import report_preferences from .preferential._system_attrs import report_skip @@ -214,6 +216,8 @@ def create_app( union_user_attrs, has_intermediate_values, ) = get_cached_extra_study_property(study_id, trials) + + plotly_graph_objects = get_plotly_graph_objects(system_attrs) return serialize_study_detail( summary, best_trials, @@ -222,6 +226,7 @@ def create_app( union, union_user_attrs, has_intermediate_values, + plotly_graph_objects, ) @app.get("/api/studies//param_importances") @@ -270,17 +275,34 @@ def create_app( @json_api_view def post_preference(study_id: int) -> dict[str, Any]: try: - best_trials = [int(d) for d in request.json.get("best_trials", [])] - worst_trials = [int(d) for d in request.json.get("worst_trials", [])] + mode = request.json.get("mode", "") + candidates = [int(d) for d in request.json.get("candidates", [])] + clicked = int(request.json.get("clicked", -1)) except ValueError: response.status = 400 - return {"reason": "best_trials and worst_trials must be an array of integers."} - if len(best_trials) == 0 or len(worst_trials) == 0: - response.status = 400 # Bad request - return {"reason": "You need to set best_trials and worst_trials"} + return { + "reason": ( + "`candidates` should be an array of integers and " + "`clicked` should be an integer." + ) + } - preferences = [(best, worst) for best in best_trials for worst in worst_trials] - report_preferences(study_id, storage, preferences) + if clicked == -1: + response.status = 400 + return {"reason": "`clicked` should be specified."} + if mode != "ChooseWorst": + response.status = 400 + return {"reason": "`mode` should be 'ChooseWorst'."} + + report_history( + study_id, + storage, + NewHistory( + mode=mode, + candidates=candidates, + clicked=clicked, + ), + ) response.status = 204 return {} diff --git a/optuna_dashboard/_custom_plot_data.py b/optuna_dashboard/_custom_plot_data.py new file mode 100644 index 00000000..a4dfa4af --- /dev/null +++ b/optuna_dashboard/_custom_plot_data.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import math +from typing import TYPE_CHECKING +import uuid + +from optuna import Study + + +if TYPE_CHECKING: + from typing import Any + + from optuna.storages import BaseStorage + import plotly.graph_objs as go + + +SYSTEM_ATTR_PLOT_DATA = "dashboard:plot_data:" +SYSTEM_ATTR_MAX_LENGTH = 2045 + + +def save_plotly_graph_object( + study: Study, figure: go.Figure, *, graph_object_id: str | None = None +) -> str: + """Save the user-defined plotly's graph object to the study. + + Example: + + .. code-block:: python + + import optuna + from optuna_dashboard import save_plotly_graph_object + + def objective(trial): + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + study.optimize(objective, n_trials=100) + + figure = optuna.visualization.plot_optimization_history(study) + save_plotly_graph_object(study, figure) + + Args: + study: + Target study object. + plot_data: + The plotly's graph object to save. + graph_object_id: + Unique identifier of the graph object. If specified, the graph object is overwritten. + This must be a valid HTML id attribute value. + + Returns: + The graph object ID. + """ + if graph_object_id is not None and not is_valid_graph_object_id(graph_object_id): + raise ValueError("graph_object_id must be a valid HTML id attribute value.") + + storage = study._storage + study_id = study._study_id + + graph_object_id = graph_object_id or str(uuid.uuid4()) + key = SYSTEM_ATTR_PLOT_DATA + graph_object_id + ":" + plot_data_json_str = figure.to_json() + save_graph_object_json(storage, study_id, key, plot_data_json_str) + return graph_object_id + + +def save_graph_object_json( + storage: BaseStorage, study_id: int, key_prefix: str, plot_data_json_str: str +) -> None: + plot_data_system_attrs = split_plot_data(plot_data_json_str, key_prefix) + for k, v in plot_data_system_attrs.items(): + storage.set_study_system_attr(study_id, k, v) + + # Clear previous graph object attributes + study_system_attrs = storage.get_study_system_attrs(study_id) + all_plot_data_system_attrs = [k for k in study_system_attrs if k.startswith(key_prefix)] + if len(all_plot_data_system_attrs) > len(plot_data_system_attrs): + for i in range(len(plot_data_system_attrs), len(all_plot_data_system_attrs)): + storage.set_study_system_attr(study_id, f"{key_prefix}{i}", "") + + +def list_graph_object_ids(system_attrs: dict[str, Any]) -> list[str]: + titles = set() + for key in system_attrs: + if not key.startswith(SYSTEM_ATTR_PLOT_DATA): + continue + + s = key.split(":", maxsplit=2) # e.g. ["dashboard", "plot_data", "Optimization History:1"] + if len(s) != 3: + continue + # Please note that title may contain ":". + title = s[2].rsplit(":", maxsplit=1)[0] + titles.add(title) + return list(titles) + + +def get_plotly_graph_objects(system_attrs: dict[str, Any]) -> dict[str, str]: + graph_objects = {} + for title in list_graph_object_ids(system_attrs): + key_prefix = SYSTEM_ATTR_PLOT_DATA + title + ":" + plot_data_attrs = {k: v for k, v in system_attrs.items() if k.startswith(key_prefix)} + graph_objects[title] = concat_plot_data(plot_data_attrs, key_prefix) + return graph_objects + + +def split_plot_data(plot_data_str: str, key_prefix: str) -> dict[str, str]: + plot_data_len = len(plot_data_str) + attrs = {} + for i in range(math.ceil(plot_data_len / SYSTEM_ATTR_MAX_LENGTH)): + start = i * SYSTEM_ATTR_MAX_LENGTH + end = min((i + 1) * SYSTEM_ATTR_MAX_LENGTH, plot_data_len) + attrs[f"{key_prefix}{i}"] = plot_data_str[start:end] + return attrs + + +def concat_plot_data(plot_data_attrs: dict[str, str], key_prefix: str) -> str: + return "".join(plot_data_attrs[f"{key_prefix}{i}"] for i in range(len(plot_data_attrs))) + + +def is_valid_graph_object_id(graph_object_id: str) -> bool: + if len(graph_object_id) == 0: + return False + + # Can only contain letters [A-Za-z], numbers [0-9], hyphens ("-"), underscores ("_"), + # colons, and periods. + if not all( + "a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9" or c in ("-", "_", ":", ".") + for c in graph_object_id[1:] + ): + return False + # Unlike HTML id attribute, graph object id can begin with a letter [A-Za-z] + return True diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py new file mode 100644 index 00000000..6b81b9bb --- /dev/null +++ b/optuna_dashboard/_preferential_history.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +import json +from typing import TYPE_CHECKING +import uuid + +from optuna.storages import BaseStorage + +from .preferential._system_attrs import report_preferences + + +_SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" + +if TYPE_CHECKING: + from typing import Literal + from typing import TypedDict + + FeedbackMode = Literal["ChooseWorst"] + ChooseWorstHistory = TypedDict( + "ChooseWorstHistory", + { + "mode": FeedbackMode, + "id": str, + "preference_id": str, + "timestamp": str, + "candidates": list[int], + "clicked": int, + }, + ) + History = ChooseWorstHistory + + +@dataclass +class NewHistory: + mode: FeedbackMode + candidates: list[int] + clicked: int + + +def report_history( + study_id: int, + storage: BaseStorage, + input_data: NewHistory, +) -> None: + preferences = [] + # TODO(moririn): Use TypeGuard after adding other history types. + if input_data.mode == "ChooseWorst": + preferences = [ + (best, input_data.clicked) + for best in input_data.candidates + if best != input_data.clicked + ] + else: + assert False, f"Unknown data: {input_data}" + + preference_id = report_preferences( + study_id=study_id, + storage=storage, + preferences=preferences, + ) + history_id = str(uuid.uuid4()) + + if input_data.mode == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": history_id, + "preference_id": preference_id, + "timestamp": datetime.now().isoformat(), + "candidates": input_data.candidates, + "clicked": input_data.clicked, + } + + key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id + storage.set_study_system_attr( + study_id=study_id, + key=key, + value=json.dumps(history), + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index b2494068..1f49e925 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import json from typing import Any from typing import TYPE_CHECKING @@ -16,6 +17,7 @@ from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -24,6 +26,9 @@ if TYPE_CHECKING: from typing import Literal from typing import TypedDict + from ._preferential_history import ChooseWorstHistory + from ._preferential_history import History + Attribute = TypedDict( "Attribute", { @@ -129,6 +134,7 @@ def serialize_study_detail( union: list[tuple[str, BaseDistribution]], union_user_attrs: list[tuple[str, bool]], has_intermediate_values: bool, + plotly_graph_objects: dict[str, str], ) -> dict[str, Any]: serialized: dict[str, Any] = { "name": summary.study_name, @@ -161,9 +167,38 @@ def serialize_study_detail( serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] + if serialized["is_preferential"]: + serialized["preference_history"] = serialize_preference_history(system_attrs) + serialized["plotly_graph_objects"] = [ + {"id": id_, "graph_object": graph_object} + for id_, graph_object in plotly_graph_objects.items() + ] return serialized +def serialize_preference_history( + system_attrs: dict[str, Any], +) -> list[History]: + histories: list[History] = [] + for k, v in system_attrs.items(): + if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): + continue + choice: dict[str, Any] = json.loads(v) + if choice["mode"] == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": choice["id"], + "preference_id": choice["preference_id"], + "timestamp": choice["timestamp"], + "candidates": choice["candidates"], + "clicked": choice["clicked"], + } + histories.append(history) + + histories.sort(key=lambda c: datetime.fromisoformat(c["timestamp"])) + return histories + + def serialize_frozen_trial( study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any] ) -> dict[str, Any]: diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index ce691f42..6e093683 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -14,6 +14,7 @@ from optuna.trial import FrozenTrial from optuna.trial import TrialState from optuna_dashboard.preferential._system_attrs import get_n_generate from optuna_dashboard.preferential._system_attrs import get_preferences +from optuna_dashboard.preferential._system_attrs import get_skipped_trial_ids from optuna_dashboard.preferential._system_attrs import is_skipped_trial from optuna_dashboard.preferential._system_attrs import report_preferences from optuna_dashboard.preferential._system_attrs import set_n_generate @@ -21,7 +22,6 @@ from optuna_dashboard.preferential._system_attrs import set_n_generate _logger = logging.get_logger(__name__) _SYSTEM_ATTR_PREFERENTIAL_STUDY = "preference:is_preferential" -_SYSTEM_ATTR_COMPARISON_READY = "preference:comparison_ready" class PreferentialStudy: @@ -62,13 +62,6 @@ class PreferentialStudy: def best_trials(self) -> list[FrozenTrial]: """Return the trials that is not dominated by other trials. - .. seealso:: - - See `Study.best_trials`_ for details. - - .. _Study.best_trials: https://optuna.readthedocs.io/en/stable/reference/\ - generated/optuna.study.Study.html#optuna.study.Study.best_trials - Returns: A list of FrozenTrial object """ @@ -182,6 +175,38 @@ class PreferentialStudy: """ self._study.add_trials(trials) + def enqueue_trial( + self, + params: dict[str, Any], + user_attrs: dict[str, Any] | None = None, + skip_if_exists: bool = False, + ) -> None: + """Enqueue a trial with given parameter values. + + You can fix the next sampling parameters which will be evaluated in your + objective function. + + .. seealso:: + + See `Study.enqueue_trials`_ for details. + + .. _Study.get_trials: https://optuna.readthedocs.io/en/stable/reference/\ + generated/optuna.study.Study.html#optuna.study.Study.enqueue_trials + + Args: + params: + Parameter values to pass your objective function. + user_attrs: + A dictionary of user-specific attributes other than ``params``. + skip_if_exists: + When :obj:`True`, prevents duplicate trials from being enqueued again. + + .. note:: + This method might produce duplicated trials if called simultaneously + by multiple processes at the same time with same ``params`` dict. + """ + self._study.enqueue_trial(params, user_attrs, skip_if_exists) + def report_preference( self, better_trials: FrozenTrial | list[FrozenTrial], @@ -219,8 +244,11 @@ class PreferentialStudy: Returns: A list of the pair of FrozenTrial objects. The left trial is better than the right one. """ + + preferences = get_preferences( + self._study._storage.get_study_system_attrs(self._study._study_id) + ) # Must come before study.get_trials() trials = self._study.get_trials(deepcopy=deepcopy) - preferences = get_preferences(self._study._study_id, self._study._storage) return [(trials[better], trials[worse]) for (better, worse) in preferences] def set_user_attr(self, key: str, value: Any) -> None: @@ -237,24 +265,6 @@ class PreferentialStudy: """ self._study.set_user_attr(key, value) - def mark_comparison_ready(self, trial_or_number: optuna.Trial | int) -> None: - """Mark trials ready to compare. - - Args: - trial_or_number: - A Trial object or trial_number. - """ - storage = self._study._storage - if isinstance(trial_or_number, optuna.Trial): - trial_id = trial_or_number._trial_id - elif isinstance(trial_or_number, int): - trial_id = storage.get_trial_id_from_study_id_trial_number( - self._study._study_id, trial_or_number - ) - else: - raise RuntimeError("Unexpected trial type") - storage.set_trial_system_attr(trial_id, _SYSTEM_ATTR_COMPARISON_READY, True) - def should_generate(self) -> bool: """Return whether the generator should generate a new trial now. @@ -263,21 +273,33 @@ class PreferentialStudy: to generate a new trial if this method returns :obj:`True`, and to wait for human evaluation if this method returns :obj:`False`. """ - return len(self.best_trials) < get_n_generate(self._study.system_attrs) + study_system_attrs = self._study._storage.get_study_system_attrs( + self._study._study_id + ) # Must come before _study.get_trials() + trials = self._study.get_trials( + deepcopy=False, states=(TrialState.COMPLETE, TrialState.RUNNING) + ) + worse_trial_numbers = {worse for _, worse in get_preferences(study_system_attrs)} + skipped_trial_ids = set(get_skipped_trial_ids(study_system_attrs)) + active_trials = [ + t + for t in trials + if t.number not in worse_trial_numbers and t._trial_id not in skipped_trial_ids + ] + return len(active_trials) < get_n_generate(self._study.system_attrs) def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]: - preferences = get_preferences(study_id, storage) + preferences = get_preferences(storage.get_study_system_attrs(study_id)) worse_numbers = {worse for _, worse in preferences} + nondominated_numbers = {better for better, _ in preferences if better not in worse_numbers} + trials = storage.get_all_trials(study_id, deepcopy=False) + study_system_attrs = storage.get_study_system_attrs(study_id) + best_trials = [] - for t in storage.get_all_trials( - study_id, deepcopy=False, states=(TrialState.COMPLETE, TrialState.RUNNING) - ): - if not t.system_attrs.get(_SYSTEM_ATTR_COMPARISON_READY, False): - continue - if t.number in worse_numbers: - continue + for n in nondominated_numbers: + t = trials[n] if is_skipped_trial(t._trial_id, study_system_attrs): continue best_trials.append(copy.deepcopy(t)) diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 2964c9e0..47c2a486 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -16,8 +16,9 @@ def report_preferences( study_id: int, storage: BaseStorage, preferences: list[tuple[int, int]], -) -> None: - key = _SYSTEM_ATTR_PREFIX_PREFERENCE + str(uuid.uuid4()) +) -> str: + preference_id = str(uuid.uuid4()) + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_id storage.set_study_system_attr( study_id=study_id, key=key, @@ -31,15 +32,12 @@ def report_preferences( trial_id = trials[number]._trial_id if trials[number].state != TrialState.COMPLETE: storage.set_trial_state_values(trial_id, TrialState.COMPLETE, values) + return preference_id -def get_preferences( - study_id: int, - storage: BaseStorage, -) -> list[tuple[int, int]]: +def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]]: preferences: list[tuple[int, int]] = [] - system_attrs = storage.get_study_system_attrs(study_id) - for k, v in system_attrs.items(): + for k, v in study_system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_PREFERENCE): continue preferences.extend(v) # type: ignore @@ -63,6 +61,19 @@ def is_skipped_trial(trial_id: int, study_system_attrs: dict[str, Any]) -> bool: return key in study_system_attrs +def get_skipped_trial_ids(study_system_attrs: dict[str, Any]) -> list[int]: + skipped_trial_ids: list[int] = [] + for k in study_system_attrs: + if not k.startswith(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL): + continue + try: + trial_id = int(k[len(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL) :]) # noqa: E203 + skipped_trial_ids.append(trial_id) + except ValueError: + continue + return skipped_trial_ids + + def get_n_generate(study_system_attrs: dict[str, Any]) -> int: return study_system_attrs[_SYSTEM_ATTR_N_GENERATE] diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index ff4001c1..8349b1a4 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -1,155 +1,41 @@ from __future__ import annotations import math -from math import erfc from typing import Any +from typing import Callable -from botorch.acquisition.analytic import LogExpectedImprovement -from botorch.models.gpytorch import GPyTorchModel -from botorch.optim import optimize_acqf +import botorch.acquisition.analytic +import botorch.models.model +import botorch.optim +import botorch.posteriors.gpytorch import gpytorch.constraints import gpytorch.kernels -import gpytorch.likelihoods.gaussian_likelihood -from gpytorch.likelihoods.gaussian_likelihood import GaussianLikelihood -from gpytorch.likelihoods.gaussian_likelihood import Interval from gpytorch.likelihoods.gaussian_likelihood import Prior -from gpytorch.models.exact_gp import ExactGP -import gpytorch.module -from linear_operator.operators import DiagLinearOperator -from linear_operator.operators import LinearOperator -from linear_operator.utils.errors import NotPSDError import numpy as np import optuna -from optuna import distributions -from optuna import Study -from optuna._transform import _SearchSpaceTransform -from optuna.distributions import BaseDistribution -from optuna.search_space import IntersectionSearchSpace -from optuna.trial import FrozenTrial -import pyro -import pyro.infer.autoguide -import pyro.infer.mcmc -from scipy.special import erfcinv +import optuna._transform import torch from torch import Tensor from .._system_attrs import get_preferences -class _WeightedGaussianLikelihood(GaussianLikelihood): - def __init__( - self, - weights: torch.Tensor | None = None, - noise_prior: Prior | None = None, - noise_constraint: Interval | None = None, - batch_shape: torch.Size = torch.Size(), - **kwargs: Any, - ) -> None: - super().__init__( - noise_prior=noise_prior, - noise_constraint=noise_constraint, - batch_shape=batch_shape, - **kwargs, - ) - self.weights = weights - - def _shaped_noise_covar( - self, base_shape: torch.Size, *params: Any, **kwargs: Any - ) -> Tensor | LinearOperator: - assert self.weights is not None - assert base_shape[-1] == self.weights.shape[-1] - return DiagLinearOperator(1.0 / self.weights) * super()._shaped_noise_covar( - base_shape, *params, **kwargs - ) - - -def _sample_y( - preferences: np.ndarray, - cov_X_X: np.ndarray, - obs_noise_var: float, - cycles: int, - initial_sample: np.ndarray, - rng: np.random.RandomState, -) -> np.ndarray: - # TODO: Refactor and write tests for this function. - - N = cov_X_X.shape[0] - M = len(preferences) - cov_X_X = cov_X_X + np.eye(N) * 1e-6 # Add jitter - cov_X_X_chol = np.linalg.cholesky(cov_X_X) - cov_X_X_inv = np.linalg.inv(cov_X_X) - - # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T - - schur = cov_X_X_inv.copy() - np.add.at(schur, (preferences[:, 0], preferences[:, 0]), 1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 1], preferences[:, 1]), 1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 0], preferences[:, 1]), -1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 1], preferences[:, 0]), -1.0 / (2 * obs_noise_var)) - idx_M = np.arange(M) - - schur_inv = np.linalg.inv(schur) - - cov_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] - cov_diff_inv = cov_diff_inv[preferences[:, 0], :] - cov_diff_inv[preferences[:, 1], :] - cov_diff_inv *= -1 / (2 * obs_noise_var) ** 2 - cov_diff_inv[idx_M, idx_M] += 1.0 / (2 * obs_noise_var) - - diffs = _orthants_MVN_Gibbs_sampling( - cov_diff_inv, - cycles=cycles, - initial_sample=initial_sample[:, 0] - initial_sample[:, 1], - rng=rng, - )[-1] - - random_ys = (cov_X_X_chol @ rng.randn(N))[preferences] + np.sqrt(obs_noise_var) * rng.randn( - M, 2 - ) - errors = diffs - (random_ys[:, 0] - random_ys[:, 1]) - cov_diff_inv_errors = cov_diff_inv @ errors - - AT_cov_diff_inv_errors = np.zeros((N,)) - np.add.at(AT_cov_diff_inv_errors, preferences[:, 0], cov_diff_inv_errors) - np.add.at(AT_cov_diff_inv_errors, preferences[:, 1], -cov_diff_inv_errors) - - return ( - random_ys - + (cov_X_X @ AT_cov_diff_inv_errors)[preferences] - + obs_noise_var * np.array([[1, -1]]) * cov_diff_inv_errors[:, None] - ) - - -_SQRT2 = math.sqrt(2) - - -def _orthants_MVN_Gibbs_sampling( - cov_inv: np.ndarray, - cycles: int, - initial_sample: np.ndarray, - rng: np.random.RandomState, -) -> np.ndarray: +def _orthants_MVN_Gibbs_sampling(cov_inv: Tensor, cycles: int, initial_sample: Tensor) -> Tensor: dim = cov_inv.shape[0] assert cov_inv.shape == (dim, dim) - if initial_sample is None: - sample_chain = np.zeros(dim) - else: - sample_chain = initial_sample + sample_chain = initial_sample + conditional_std = torch.rsqrt(torch.diag(cov_inv)) + scaled_cov_inv = cov_inv / torch.diag(cov_inv)[:, None] - conditional_std = 1 / np.sqrt(np.diag(cov_inv)) - - scaled_cov_inv = cov_inv / np.c_[np.diag(cov_inv)] - - out = np.empty((cycles + 1, dim)) + out = torch.empty((cycles + 1, dim), dtype=torch.float64) out[0, :] = sample_chain for i in range(cycles): for j in range(dim): conditional_mean = sample_chain[j] - scaled_cov_inv[j] @ sample_chain sample_chain[j] = ( - _one_side_trunc_norm_sampling( - lower=-conditional_mean / conditional_std[j], rng=rng - ) + _one_side_trunc_norm_sampling(lower=-conditional_mean / conditional_std[j]) * conditional_std[j] + conditional_mean ) @@ -158,144 +44,234 @@ def _orthants_MVN_Gibbs_sampling( return out -def _one_side_trunc_norm_sampling(lower: float, rng: np.random.RandomState) -> float: - return erfcinv(rng.rand() * erfc(lower / _SQRT2)) * _SQRT2 +def _one_side_trunc_norm_sampling(lower: Tensor) -> Tensor: + if lower > 4.0: + r = torch.clamp_min(torch.rand(torch.Size(()), dtype=torch.float64), min=1e-300) + return (lower * lower - 2 * r.log()).sqrt() + else: + SQRT2 = math.sqrt(2) + r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) + while 1 - r == 1: + r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) + return torch.erfinv(1 - r) * SQRT2 -class _PreferentialGP(GPyTorchModel, ExactGP): - _num_outputs = 1 +_orthants_MVN_Gibbs_sampling_jit = torch.jit.script(_orthants_MVN_Gibbs_sampling) + +def _compute_cov_diff_diff_inv(preferences: Tensor, cov_x_x: Tensor, noise_var: Tensor) -> Tensor: + N = cov_x_x.shape[0] + M = preferences.shape[0] + + # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T + # (K^-1 + s^-1 A^T A)^-1 = K (I + s^-1 A^T A K)^-1 (To avoid computing K^-1) + + I_plus_sinv_AT_A_K = torch.eye(N, dtype=torch.float64) + A_K = cov_x_x[preferences[:, 0], :] - cov_x_x[preferences[:, 1], :] + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 0], A_K * (1 / noise_var)) + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K * (-1 / noise_var)) + schur_inv: Tensor = torch.linalg.solve(I_plus_sinv_AT_A_K, cov_x_x, left=False) + cov_diff_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] + cov_diff_diff_inv = ( + cov_diff_diff_inv[preferences[:, 0], :] - cov_diff_diff_inv[preferences[:, 1], :] + ) + cov_diff_diff_inv *= -1 / noise_var**2 + idx_M = torch.arange(M) + cov_diff_diff_inv[idx_M, idx_M] += 1.0 / noise_var + + return cov_diff_diff_inv + + +class _SampledGP(botorch.models.model.Model): def __init__( self, - kernel: gpytorch.kernels.Kernel, - noise_prior: Prior | None = None, - noise_constraint: Interval | None = None, + kernel_func: Callable[[Tensor, Tensor], Tensor], + x: Tensor, + preferences: Tensor, + noise_var: Tensor, + diff: Tensor, ) -> None: - GPyTorchModel.__init__(self) - likelihood = _WeightedGaussianLikelihood( - noise_prior=noise_prior, noise_constraint=noise_constraint + super().__init__() + self.kernel_func = kernel_func + self.x = x + self.preferences = preferences + self.diff = diff + self.noise_var = noise_var + self._cov_diff_diff_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=self.kernel_func(x, x), + noise_var=noise_var, ) - ExactGP.__init__(self, train_inputs=None, train_targets=None, likelihood=likelihood) - self.covar_module = kernel - self._last_params: dict[str, torch.Tensor] | None = None - self._last_mcmc_step_size: float | None = None + def posterior( + self, + X: Tensor, + output_indices: list[int] | None = None, + observation_noise: bool = False, + posterior_transform: Any | None = None, + **kwargs: Any, + ) -> botorch.posteriors.gpytorch.GPyTorchPosterior: + assert posterior_transform is None + assert output_indices is None + assert self.x.shape[-1] == X.shape[-1] - def _pyro_model(self, train_x: torch.Tensor, train_y: torch.Tensor) -> None: - # with gpytorch.settings.fast_computations(False, False, False): - sampled_model = self.pyro_sample_from_prior() + x_expanded = self.x.expand(X.shape[:-2] + (self.x.shape[-2], X.shape[-1])) - ys = sampled_model.likelihood(sampled_model.forward(train_x)) + cov_X_x = self.kernel_func(X, x_expanded) + cov_X_diff = cov_X_x[..., self.preferences[:, 0]] - cov_X_x[..., self.preferences[:, 1]] - pyro.sample("y", ys, obs=train_y) + mean = cov_X_diff @ (self._cov_diff_diff_inv @ self.diff) + cov = self.kernel_func(X, X) - cov_X_diff @ self._cov_diff_diff_inv @ cov_X_diff.transpose( + -1, -2 + ) + if observation_noise: + idx = torch.arange(cov.shape[-1]) + cov[..., idx, idx] += self.noise_var - def fit_mcmc( - self, X: torch.Tensor, preferences: torch.Tensor, cycles: int, rng: np.random.RandomState - ) -> None: + return botorch.posteriors.gpytorch.GPyTorchPosterior( + distribution=gpytorch.distributions.MultivariateNormal( + mean=mean, + covariance_matrix=cov, + ) + ) + + @property + def batch_shape(self) -> torch.Size: + return torch.Size() + + @property + def num_outputs(self) -> int: + return 1 + + +def _truncnorm_mean_var_logz(alpha: Tensor) -> tuple[Tensor, Tensor, Tensor]: + SQRT_HALF = math.sqrt(0.5) + SQRT_HALF_PI = math.sqrt(0.5 * math.pi) + logz = torch.special.log_ndtr(-alpha) + mean = 1 / (SQRT_HALF_PI * torch.special.erfcx(alpha * SQRT_HALF)) + var = 1 - mean * (mean - alpha) + return mean, var, logz + + +def _orthants_MVN_EP( + cov0: Tensor, preferences: Tensor, noise_var: Tensor, cycles: int +) -> tuple[Tensor, Tensor, Tensor]: + N = cov0.shape[0] + M = preferences.shape[0] + mu = torch.zeros(N, dtype=cov0.dtype) + cov = cov0.clone() + virtual_obs_a = [torch.tensor(0.0, dtype=cov0.dtype) for _ in range(M)] + virtual_obs_b = [torch.tensor(0.0, dtype=cov0.dtype) for _ in range(M)] + log_zs = torch.zeros(M, dtype=cov0.dtype) + + for _ in range(cycles): + for i in range(M): + pref_i = preferences[i, :] + mean1 = mu[pref_i[0]] - mu[pref_i[1]] + Sxy = cov[pref_i[0]] - cov[pref_i[1]] + var1 = Sxy[pref_i[0]] - Sxy[pref_i[1]] + + r0 = (1 - var1 * virtual_obs_a[i]).reciprocal() + var0 = var1 * r0 + mean0 = (mean1 + var1 * virtual_obs_b[i]) * r0 + + obs_var = var0 + noise_var + obs_sigma = torch.sqrt(obs_var) + alpha = -mean0 / torch.clamp_min(obs_sigma, min=1e-20) + mean_norm, var_norm, logz = _truncnorm_mean_var_logz(alpha) + + kalman_factor = var0 / torch.clamp_min(obs_var, min=1e-20) + mean2 = mean0 + obs_sigma * mean_norm * kalman_factor + var2 = kalman_factor * (noise_var + var_norm * var0) + + var1_var2_inv = torch.clamp_min(var1 * var2, min=1e-20).reciprocal() + db = (mean1 * var2 - mean2 * var1) * var1_var2_inv + da = (var1 - var2) * var1_var2_inv + virtual_obs_b[i] = virtual_obs_b[i] + db + virtual_obs_a[i] = virtual_obs_a[i] + da + + dr = (1 + var1 * da).reciprocal() + mu = mu - Sxy * ((db + mean1 * da) * dr) + cov = cov - (Sxy[:, None] * (da * dr)) @ Sxy[None, :] + log_zs[i] = logz + return mu, cov, torch.sum(log_zs) + + +_orthants_MVN_EP_jit = torch.jit.script(_orthants_MVN_EP) + + +class _PreferentialGP: + def __init__(self, kernel: gpytorch.kernels.Kernel, noise_prior: Prior, dims: int) -> None: + self.kernel = kernel + self.noise_prior = noise_prior + self.dims = dims + + self.diff = torch.empty((0,), dtype=torch.float64, requires_grad=False) + self.log_noise = torch.nn.Parameter( + torch.tensor(0.0, dtype=torch.float64), requires_grad=True + ) + + def fit_params_EP(self, X: Tensor, preferences: Tensor) -> None: if len(preferences) == 0: - # Skip actual MCMC computation - self.set_train_data( - inputs=torch.empty((0, X.shape[-1])), - targets=torch.empty((0,)), - strict=False, - ) - self.likelihood.weights = torch.empty((0,)) - else: - dtype = torch.float64 + return + tolerance = 1e-3 + max_iter = 100 - cnt = torch.bincount(preferences.reshape(-1)) - mask = cnt > 0 - train_x = X[mask] - weights = cnt[mask] + optim = torch.optim.LBFGS([*self.kernel.parameters(), self.log_noise]) - assert isinstance(self.likelihood, _WeightedGaussianLikelihood) - self.likelihood.weights = weights + last_params = [p.detach().clone() for p in optim.param_groups[0]["params"]] + for _ in range(max_iter): - preferences_np = preferences.detach().numpy() + def closure() -> Tensor: + optim.zero_grad() + noise = self.log_noise.exp() + cov0 = self.kernel.forward(X, X).to_dense() + _, _, logz = _orthants_MVN_EP_jit(cov0, preferences, noise, cycles=2) - all_ys_np = np.zeros((len(preferences), 2)) - train_y = torch.zeros( - ( - len( - train_x, - ) - ), - dtype=dtype, + loss = -logz - self.noise_prior.log_prob(noise) + for _, _, prior, param, _ in self.kernel.named_priors(): + loss = loss - prior.log_prob(param(self.kernel)).sum() + + loss.backward() + return loss + + optim.step(closure) + + # Check for convergence + params = optim.param_groups[0]["params"] + for p_old, p_new in zip(last_params, params): + if torch.max(torch.abs(p_old - p_new)) > tolerance: + break + else: + break + last_params = [p.detach().clone() for p in params] + + def sample_gp(self, x: Tensor, preferences: Tensor) -> _SampledGP: + self.fit_params_EP(x, preferences) + + with torch.no_grad(): + cov_diff_diff_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=self.kernel(x, x).to_dense(), + noise_var=self.log_noise.exp(), ) - nuts = pyro.infer.mcmc.NUTS( - model=self._pyro_model, - init_strategy=pyro.infer.autoguide.init_to_sample, - step_size=self._last_mcmc_step_size or 1.0, + original_diff_size = len(self.diff) + self.diff.resize_(len(preferences)) + self.diff[original_diff_size:] = 0.0 + + self.diff = _orthants_MVN_Gibbs_sampling_jit( + cov_inv=cov_diff_diff_inv, + initial_sample=self.diff, + cycles=20, + )[-1] + return _SampledGP( + kernel_func=lambda x1, x2: self.kernel(x1, x2).to_dense(), + x=x, + preferences=preferences, + noise_var=self.log_noise.exp(), + diff=self.diff, ) - warmup_steps = max(0, cycles - 2) - nuts.setup(warmup_steps=warmup_steps, train_x=train_x, train_y=train_y) - - raw_params = self._last_params or nuts.initial_params - for i in range(cycles): - params = { - name: nuts.transforms[name].inv(value) for name, value in raw_params.items() - } - _set_params(self, params) - self.set_train_data(train_x, train_y, strict=False) - all_ys_np = _sample_y( - preferences=preferences_np, - cov_X_X=self.covar_module(train_x).detach().numpy(), - obs_noise_var=float(self.likelihood.noise_covar.noise), - cycles=10, - initial_sample=all_ys_np, - rng=rng, - ) - ys_sum_np = np.zeros((len(X),)) - np.add.at(ys_sum_np, preferences_np.reshape(-1), all_ys_np.reshape(-1)) - ys_sum = torch.from_numpy(ys_sum_np) - train_y[:] = ys_sum[mask] / cnt[mask] - nuts.clear_cache() - try: - raw_params = nuts.sample(raw_params) - except NotPSDError: - nuts.cleanup() - nuts = pyro.infer.mcmc.NUTS( - model=self._pyro_model, - init_strategy=pyro.infer.autoguide.init_to_sample, - step_size=self._last_mcmc_step_size or 1.0, - ) - nuts.setup(warmup_steps=warmup_steps, train_x=train_x, train_y=train_y) - raw_params = nuts.initial_params - - params = {name: nuts.transforms[name].inv(value) for name, value in raw_params.items()} - self.set_train_data(train_x, train_y, strict=False) - _set_params(self, params) - - self._last_params = raw_params - self._last_mcmc_step_size = nuts.step_size - nuts.cleanup() - - def forward(self, x: torch.Tensor) -> gpytorch.distributions.MultivariateNormal: - mean_module = gpytorch.means.ZeroMean() - return gpytorch.distributions.MultivariateNormal( - mean_module(x), - self.covar_module(x), - ) - - -def _set_params( - module: gpytorch.Module, - params_dict: dict[str, torch.Tensor], - memo: set | None = None, - prefix: str = "", -) -> None: - if memo is None: - memo = set() - if hasattr(module, "_priors"): - for name, (prior, closure, setting_closure) in module._priors.items(): - if prior is not None and prior not in memo: - memo.add(prior) - setting_closure(module, params_dict[prefix + ("." if prefix else "") + name]) - - for mname, module_ in module.named_children(): - submodule_prefix = prefix + ("." if prefix else "") + mname - _set_params(module_, params_dict, memo=memo, prefix=submodule_prefix) class PreferentialGPSampler(optuna.samplers.BaseSampler): @@ -306,18 +282,16 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): noise_prior: Prior | None = None, independent_sampler: optuna.samplers.BaseSampler | None = None, seed: int | None = None, - device: torch.device | None = None, ) -> None: - self._rng = np.random.RandomState(seed) - self._search_space = IntersectionSearchSpace() - self.kernel = kernel - self.noise_prior = noise_prior - self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( - seed=self._rng.randint(2**32), - ) - self.device = device or torch.device("cpu") + self.noise_prior = noise_prior or gpytorch.priors.GammaPrior(5.0, 50.0) + self._rng = np.random.RandomState(seed) + self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( + seed=self._rng.randint(2**32) + ) + + self._search_space = optuna.search_space.IntersectionSearchSpace() self._gp: _PreferentialGP | None = None def reseed_rng(self) -> None: @@ -325,75 +299,64 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): self._rng = np.random.RandomState() def infer_relative_search_space( - self, study: Study, trial: FrozenTrial - ) -> dict[str, BaseDistribution]: + self, study: optuna.Study, trial: optuna.trial.FrozenTrial + ) -> dict[str, optuna.distributions.BaseDistribution]: return self._search_space.calculate(study) def sample_relative( self, - study: Study, - trial: FrozenTrial, - search_space: dict[str, BaseDistribution], + study: optuna.Study, + trial: optuna.trial.FrozenTrial, + search_space: dict[str, optuna.distributions.BaseDistribution], ) -> dict[str, Any]: + preferences = get_preferences(study.system_attrs) + if len(preferences) == 0: + return {} + + trials = study.get_trials(deepcopy=False) + trials_with_preference = list({t for (b, w) in preferences for t in (b, w)}) + ids = {t: i for i, t in enumerate(trials_with_preference)} + + trans = optuna._transform._SearchSpaceTransform( + search_space, transform_log=True, transform_step=True, transform_0_1=True + ) + params = torch.tensor( + np.array([trans.transform(trials[t].params) for t in trials_with_preference]), + dtype=torch.float64, + ) + pref_ids = torch.tensor([[ids[b], ids[w]] for b, w in preferences], dtype=torch.int32) with torch.random.fork_rng(): torch.manual_seed(self._rng.randint(2**32)) - pyro.set_rng_seed(self._rng.randint(2**32)) - if len(search_space) == 0: - return {} - - preferences = get_preferences(study._study_id, study._storage) - trials = study.get_trials(deepcopy=False) - if len(preferences) == 0: - return {} - - trans = _SearchSpaceTransform( - search_space, transform_log=True, transform_step=True, transform_0_1=True - ) - dims = len(trans.bounds) self._gp = self._gp or _PreferentialGP( kernel=self.kernel or gpytorch.kernels.MaternKernel( - nu=2.5, - ard_num_dims=dims, - lengthscale_prior=gpytorch.priors.GammaPrior(3.0, 6.0), - lengthscale_constraint=gpytorch.constraints.Positive(), + nu=1.5, + ard_num_dims=len(trans.bounds), + lengthscale_prior=gpytorch.priors.GammaPrior(5.0, 10.0), + lengthscale_constraint=gpytorch.constraints.GreaterThan( + 0.0, + transform=torch.exp, + inv_transform=torch.log, + ), ), - noise_prior=self.noise_prior or gpytorch.priors.GammaPrior(1.1, 2.0), - noise_constraint=gpytorch.constraints.Positive(), + noise_prior=self.noise_prior, + dims=len(trans.bounds), ) + if self._gp.dims != len(trans.bounds): + raise NotImplementedError( + "The search space has changed. " + "Dynamic search space is not supported in PreferentialGPSampler." + ) - ids: dict[int, int] = {} - params: list[torch.Tensor] = [] - pref_ids: list[tuple[int, int]] = [] - - for better, worse in preferences: - for t in (better, worse): - if t not in ids: - ids[t] = len(ids) - params.append(trans.transform(trials[t].params)) - pref_ids.append((ids[better], ids[worse])) - dtype = torch.float64 - - params_torch = torch.tensor(np.array(params), dtype=dtype, device=self.device) - pref_ids_torch = torch.tensor( - np.array(pref_ids), - dtype=torch.int32, - device=self.device, - ) - self._gp.fit_mcmc(params_torch, pref_ids_torch, cycles=10, rng=self._rng) - self._gp.eval() - scores = self._gp(params_torch).mean - - best_f = torch.max(scores) - - acqf = LogExpectedImprovement( - model=self._gp, - best_f=best_f, + sampled_gp = self._gp.sample_gp(params, pref_ids) + acqf = botorch.acquisition.analytic.LogExpectedImprovement( + model=sampled_gp, + best_f=torch.max(sampled_gp.posterior(params[:, None, :]).mean), ) # TODO: Make it possible to apply it on categorical variables - candidates, _ = optimize_acqf( + candidates, _ = botorch.optim.optimize_acqf( acq_function=acqf, bounds=torch.from_numpy(trans.bounds.T), q=1, @@ -407,10 +370,10 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): def sample_independent( self, - study: Study, - trial: FrozenTrial, + study: optuna.Study, + trial: optuna.trial.FrozenTrial, param_name: str, - param_distribution: distributions.BaseDistribution, + param_distribution: optuna.distributions.BaseDistribution, ) -> Any: return self.independent_sampler.sample_independent( study, trial, param_name, param_distribution diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index f0c82e2b..8979d3e3 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -588,10 +588,10 @@ export const actionCreator = () => { const updatePreference = ( study_id: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ) => { - reportPreferenceAPI(study_id, best_trials, worst_trials).catch((err) => { + reportPreferenceAPI(study_id, candidates, clicked).catch((err) => { const reason = err.response?.data.reason enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { variant: "error", diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index caa57716..6ae34403 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -55,6 +55,28 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } +interface PreferenceHistoryResponce { + id: string + preference_id: string + candidates: number[] + clicked: number + mode: PreferenceFeedbackMode + timestamp: string +} + +const convertPreferenceHistory = ( + res: PreferenceHistoryResponce +): PreferenceHistory => { + return { + id: res.id, + preference_id: res.preference_id, + candidates: res.candidates, + clicked: res.clicked, + feedback_mode: res.mode, + timestamp: new Date(res.timestamp), + } +} + interface StudyDetailResponse { name: string datetime_start: string @@ -70,8 +92,8 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets - feedback_component_type?: string - feedback_artifact_key?: string + preference_history?: PreferenceHistoryResponce[] + plotly_graph_objects: PlotlyGraphObject[] } export const getStudyDetailAPI = ( @@ -110,6 +132,10 @@ export const getStudyDetailAPI = ( feedback_component_type: res.data .feedback_component_type as FeedbackComponentType, feedback_artifact_key: res.data.feedback_artifact_key, + preference_history: res.data.preference_history?.map( + convertPreferenceHistory + ), + plotly_graph_objects: res.data.plotly_graph_objects, } }) } @@ -319,13 +345,14 @@ export const getParamImportances = ( export const reportPreferenceAPI = ( studyId: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ): Promise => { return axiosInstance .post(`/api/studies/${studyId}/preference`, { - best_trials: best_trials, - worst_trials: worst_trials, + candidates: candidates, + clicked: clicked, + mode: "ChooseWorst", }) .then(() => { return diff --git a/optuna_dashboard/ts/components/App.tsx b/optuna_dashboard/ts/components/App.tsx index 78015049..8adf8895 100644 --- a/optuna_dashboard/ts/components/App.tsx +++ b/optuna_dashboard/ts/components/App.tsx @@ -96,6 +96,15 @@ export const App: FC = () => { /> } /> + + } + /> } diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 0903d72b..446b362e 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -34,12 +34,19 @@ import GitHubIcon from "@mui/icons-material/GitHub" import OpenInNewIcon from "@mui/icons-material/OpenInNew" import QueryStatsIcon from "@mui/icons-material/QueryStats" import ThumbUpAltIcon from "@mui/icons-material/ThumbUpAlt" +import HistoryIcon from "@mui/icons-material/History" import { Switch } from "@mui/material" import { actionCreator } from "../action" const drawerWidth = 240 -export type PageId = "top" | "analytics" | "trialTable" | "trialList" | "note" +export type PageId = + | "top" + | "analytics" + | "trialTable" + | "trialList" + | "note" + | "preferenceHistory" const openedMixin = (theme: Theme): CSSObject => ({ width: drawerWidth, @@ -204,6 +211,28 @@ export const AppDrawer: FC<{ /> + {isPreferential && ( + + + + + + + + + )} = ({ artifact, urlPath, height }) => { + if (isThreejsArtifact(artifact)) { + return ( + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + ) + } else if (artifact.mimetype.startsWith("image")) { + return ( + + ) + } + return +} diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx new file mode 100644 index 00000000..3bc5a750 --- /dev/null +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -0,0 +1,218 @@ +import React, { FC, useState } from "react" +import { + Typography, + Box, + useTheme, + Card, + CardContent, + CardActions, +} from "@mui/material" +import ClearIcon from "@mui/icons-material/Clear" +import IconButton from "@mui/material/IconButton" +import OpenInFullIcon from "@mui/icons-material/OpenInFull" +import Modal from "@mui/material/Modal" +import { red } from "@mui/material/colors" + +import { TrialListDetail } from "./TrialList" +import { MarkdownRenderer } from "./Note" +import { formatDate } from "../dateUtil" + +type TrialType = "worst" | "none" + +const CandidateTrial: FC<{ + trial: Trial + type: TrialType +}> = ({ trial, type }) => { + const theme = useTheme() + const trialWidth = 300 + const trialHeight = 300 + const [detailShown, setDetailShown] = useState(false) + + const cardComponentSx = { + padding: 0, + position: "relative", + overflow: "hidden", + "::before": {}, + } + if (type !== "none") { + cardComponentSx["::before"] = { + content: '""', + position: "absolute", + top: 0, + left: 0, + width: "100%", + height: "100%", + backgroundColor: theme.palette.mode === "dark" ? "white" : "black", + opacity: 0.2, + zIndex: 1, + transition: "opacity 0.3s ease-out", + } + } + + return ( + + + Trial {trial.number} + setDetailShown(true)} + aria-label="show detail" + > + + + + + + + + + {type === "worst" ? ( + + ) : null} + + setDetailShown(false)}> + + + false} + directions={[]} + objectiveNames={[]} + /> + + + + + ) +} + +const ChoiceTrials: FC<{ choice: PreferenceHistory; trials: Trial[] }> = ({ + choice, + trials, +}) => { + const theme = useTheme() + const worst_trials = new Set([choice.clicked]) + + return ( + + + {formatDate(choice.timestamp)} + + + {choice.candidates.map((trial_num, index) => ( + + ))} + + + ) +} + +export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({ + studyDetail, +}) => { + if ( + studyDetail === null || + !studyDetail.is_preferential || + studyDetail.preference_history === undefined + ) { + return null + } + const theme = useTheme() + const preference_histories = [...studyDetail.preference_history] + + if (preference_histories.length === 0) { + return ( + + No feedback history + + ) + } + + return ( + + {preference_histories.reverse().map((choice) => ( + + ))} + + ) +} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index ec597a09..e68e7ec9 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -13,46 +13,23 @@ import { FormLabel, Modal, } from "@mui/material" +import IconButton from "@mui/material/IconButton" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import ReplayIcon from "@mui/icons-material/Replay" import ClearIcon from "@mui/icons-material/Clear" -import IconButton from "@mui/material/IconButton" import SettingsIcon from "@mui/icons-material/Settings" +import FullscreenIcon from "@mui/icons-material/Fullscreen" import red from "@mui/material/colors/red" + import { actionCreator } from "../action" -import { MarkdownRenderer } from "./Note" +import { TrialListDetail } from "./TrialList" import { - TrialArtifactActions, - TrialArtifactContent, - TrialListDetail, -} from "./TrialList" - -const FeedbackContent: FC<{ - trial: Trial - artifact?: Artifact - componentId: FeedbackComponentType - width: string - minHeight: string -}> = ({ trial, artifact, componentId, width, minHeight }) => { - if (componentId === "Note") { - return - } - if (componentId === "Artifact") { - if (artifact === undefined) { - return null - } - return ( - - ) - } - - return null -} + isThreejsArtifact, + useThreejsArtifactModal, +} from "./ThreejsArtifactViewer" +import { ArtifactCardMedia } from "./ArtifactCardMedia" +import { MarkdownRenderer } from "./Note" +import { Details } from "@mui/icons-material" const ModalPage: FC<{ children: React.ReactNode @@ -73,7 +50,7 @@ const ModalPage: FC<{ maxHeight: "90%", margin: "auto", overflow: "hidden", - backgroundColor: theme.palette.mode === "dark" ? "black" : "white", + backgroundColor: theme.palette.background.default, borderRadius: theme.spacing(3), }} > @@ -91,177 +68,6 @@ const ModalPage: FC<{ ) } -const PreferentialTrial: FC<{ - trial?: Trial - studyDetail: StudyDetail - hideTrial: () => void -}> = ({ trial, studyDetail, hideTrial }) => { - const theme = useTheme() - const action = actionCreator() - const trialWidth = 400 - const trialHeight = 300 - const [detailShown, setDetailShown] = useState(false) - const [buttonHover, setButtonHover] = useState(false) - const componentId = studyDetail.feedback_component_type ?? "Note" - const artifactKey = studyDetail.feedback_artifact_key - const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value - const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) - - if (trial == undefined) { - return ( - - ) - } - const onFeedback = () => { - hideTrial() - const best_trials = studyDetail.best_trials - .map((t) => t.number) - .filter((t) => t !== trial.number) - action.updatePreference(trial.study_id, best_trials, [trial.number]) - } - - return ( - - - Trial {trial.number} - {componentId === "Artifact" && artifact !== undefined ? ( - - {`(${artifact.filename})`} - - ) : null} - {componentId === "Artifact" && artifact !== undefined ? ( - - ) : null} - { - hideTrial() - action.skipPreferentialTrial(trial.study_id, trial.trial_id) - }} - aria-label="skip trial" - > - - - setDetailShown(true)} - aria-label="show detail" - > - - - - { - if (e.shiftKey) onFeedback() - }} - sx={{ - position: "relative", - padding: theme.spacing(2), - overflow: "hidden", - minHeight: theme.spacing(20), - }} - > - - - - - - - - { - setDetailShown(false) - }} - > - true} - directions={[]} - objectiveNames={[]} - /> - - - ) -} - const SettingsPage: FC<{ studyDetail: StudyDetail settingShown: boolean @@ -358,6 +164,29 @@ const SettingsPage: FC<{ ) } +const FeedbackContent: FC<{ + trial: Trial + artifact?: Artifact + componentId: FeedbackComponentType + width: string + minHeight: string + urlPath: string +}> = ({ trial, artifact, componentId, width, minHeight, urlPath }) => { + if (componentId === "Note") { + return + } + if (componentId === "Artifact") { + if (artifact === undefined) { + return null + } + return ( + + ) + } + + return null +} + type DisplayTrials = { numbers: number[] last_number: number @@ -366,16 +195,29 @@ type DisplayTrials = { export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail, }) => { + const theme = useTheme() + const action = actionCreator() + const [openThreejsArtifactModal, renderThreejsArtifactModal] = + useThreejsArtifactModal() + const runningTrials = + studyDetail?.trials.filter((t) => t.state === "Running") ?? [] + const activeTrials = runningTrials.concat(studyDetail?.best_trials ?? []) + + const [displayTrials, setDisplayTrials] = useState({ + numbers: activeTrials.map((t) => t.number), + last_number: Math.max(...activeTrials.map((t) => t.number), -1), + }) + const [settingShown, setSettingShown] = useState(false) + const [detailTrial, setDetailTrial] = useState(null) + const [buttonHover, setButtonHover] = useState(null) + + const trialWidth = 400 + const trialHeight = 300 + if (studyDetail === null || !studyDetail.is_preferential) { return null } - const theme = useTheme() - const [displayTrials, setDisplayTrials] = useState({ - numbers: studyDetail.best_trials.map((t) => t.number), - last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1), - }) - const [settingShown, setSettingShown] = useState(false) - const new_trails = studyDetail.best_trials.filter( + const new_trails = activeTrials.filter( (t) => displayTrials.last_number < t.number && displayTrials.numbers.find((n) => n === t.number) === undefined @@ -441,22 +283,209 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ Which trial is the worst? - {displayTrials.numbers.map((t, index) => ( - trial.number === t)} - studyDetail={studyDetail} - hideTrial={() => { - hideTrial(t) - }} - /> - ))} + {displayTrials.numbers.map((t, index) => { + const trial = activeTrials.find((trial) => trial.number === t) + const candidates = displayTrials.numbers.filter((n) => n !== -1) + const componentId = studyDetail.feedback_component_type ?? "Note" + const artifactKey = studyDetail.feedback_artifact_key + const artifactId = trial?.user_attrs.find( + (a) => a.key === artifactKey + )?.value + const artifact = trial?.artifacts.find( + (a) => a.artifact_id === artifactId + ) + const urlPath = `/artifacts/${studyDetail.id}/${trial?.trial_id}/${artifact?.artifact_id}` + + if (trial == undefined) { + return ( + + ) + } + + const is3dModel = + componentId === "Artifact" && + artifact !== undefined && + isThreejsArtifact(artifact) + const onFeedback = () => { + hideTrial(trial.number) + action.updatePreference(trial.study_id, candidates, trial.number) + } + + return ( + + + + Trial {trial.number} + {componentId === "Artifact" && artifact !== undefined ? ( + + {`(${artifact.filename})`} + + ) : null} + + {is3dModel ? ( + { + openThreejsArtifactModal(urlPath, artifact) + }} + > + + + ) : null} + { + hideTrial(trial.number) + action.skipPreferentialTrial(trial.study_id, trial.trial_id) + }} + aria-label="skip trial" + > + + + setDetailTrial(trial.number)} + aria-label="show detail" + > + + + + { + if (e.shiftKey) onFeedback() + }} + sx={{ + position: "relative", + padding: theme.spacing(2), + overflow: "hidden", + minHeight: theme.spacing(20), + }} + > + + + + + + + ) + })} + {detailTrial !== null && ( + { + setDetailTrial(null) + }} + > + + studyDetail.trials.find((t) => t.trial_id === trialId)?.state === + "Complete" ?? false + } + directions={[]} + objectiveNames={[]} + /> + + )} + {renderThreejsArtifactModal()} ) } diff --git a/optuna_dashboard/ts/components/StudyDetail.tsx b/optuna_dashboard/ts/components/StudyDetail.tsx index fedfd625..15cb7cf8 100644 --- a/optuna_dashboard/ts/components/StudyDetail.tsx +++ b/optuna_dashboard/ts/components/StudyDetail.tsx @@ -30,6 +30,7 @@ import { GraphEdf } from "./GraphEdf" import { TrialList } from "./TrialList" import { StudyHistory } from "./StudyHistory" import { PreferentialTrials } from "./PreferentialTrials" +import { PreferenceHistory } from "./PreferenceHistory" import { PreferentialAnalytics } from "./PreferentialAnalytics" interface ParamTypes { @@ -175,6 +176,8 @@ export const StudyDetail: FC<{ /> ) + } else if (page == "preferenceHistory") { + content = } const toolbar = ( diff --git a/optuna_dashboard/ts/components/StudyHistory.tsx b/optuna_dashboard/ts/components/StudyHistory.tsx index b47c557a..907acd1a 100644 --- a/optuna_dashboard/ts/components/StudyHistory.tsx +++ b/optuna_dashboard/ts/components/StudyHistory.tsx @@ -15,6 +15,7 @@ import { GraphIntermediateValues } from "./GraphIntermediateValues" import Grid2 from "@mui/material/Unstable_Grid2" import { DataGrid, DataGridColumn } from "./DataGrid" import { GraphHyperparameterImportance } from "./GraphHyperparameterImportances" +import { UserDefinedPlot } from "./UserDefinedPlot" import { BestTrialsCard } from "./BestTrialsCard" import { useStudyDetailValue, @@ -124,6 +125,16 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => { + {studyDetail !== null && + studyDetail.plotly_graph_objects.map((go) => ( + + + + + + + + ))} diff --git a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx index 536f1d91..a88b9d2c 100644 --- a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx +++ b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx @@ -1,10 +1,17 @@ import * as THREE from "three" -import React, { useEffect, useState } from "react" +import React, { useEffect, useState, ReactNode } from "react" import { Canvas } from "@react-three/fiber" import { GizmoHelper, GizmoViewport, OrbitControls } from "@react-three/drei" import { STLLoader } from "three/examples/jsm/loaders/STLLoader" import { Rhino3dmLoader } from "three/examples/jsm/loaders/3DMLoader" import { PerspectiveCamera } from "three" +import { Modal, Box } from "@mui/material" + +export const isThreejsArtifact = (artifact: Artifact): boolean => { + return ( + artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") + ) +} interface ThreejsArtifactViewerProps { src: string @@ -109,3 +116,48 @@ export const ThreejsArtifactViewer: React.FC = ( ) } + +export const useThreejsArtifactModal = (): [ + (path: string, artifact: Artifact) => void, + () => ReactNode +] => { + const [open, setOpen] = useState(false) + const [target, setTarget] = useState<[string, Artifact | null]>(["", null]) + + const openModal = (artifactUrlPath: string, artifact: Artifact) => { + setTarget([artifactUrlPath, artifact]) + setOpen(true) + } + + const renderDeleteStudyDialog = () => { + return ( + { + setOpen(false) + setTarget(["", null]) + }} + > + + + + + ) + } + return [openModal, renderDeleteStudyDialog] +} diff --git a/optuna_dashboard/ts/components/TrialArtifactCards.tsx b/optuna_dashboard/ts/components/TrialArtifactCards.tsx new file mode 100644 index 00000000..3e15f7ed --- /dev/null +++ b/optuna_dashboard/ts/components/TrialArtifactCards.tsx @@ -0,0 +1,233 @@ +import React, { + ChangeEventHandler, + DragEventHandler, + FC, + MouseEventHandler, + useRef, + useState, +} from "react" +import { + Typography, + Box, + useTheme, + IconButton, + Card, + CardContent, + CardActionArea, +} from "@mui/material" +import UploadFileIcon from "@mui/icons-material/UploadFile" +import DownloadIcon from "@mui/icons-material/Download" +import DeleteIcon from "@mui/icons-material/Delete" +import FullscreenIcon from "@mui/icons-material/Fullscreen" + +import { actionCreator } from "../action" +import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" +import { + useThreejsArtifactModal, + isThreejsArtifact, +} from "./ThreejsArtifactViewer" +import { ArtifactCardMedia } from "./ArtifactCardMedia" + +export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => { + const theme = useTheme() + const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = + useDeleteArtifactDialog() + const [openThreejsArtifactModal, renderThreejsArtifactModal] = + useThreejsArtifactModal() + + const width = "200px" + const height = "150px" + + return ( + <> + + Artifacts + + + {trial.artifacts.map((artifact) => { + const urlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${artifact.artifact_id}` + return ( + + + + + {artifact.filename} + + {isThreejsArtifact(artifact) ? ( + { + openThreejsArtifactModal(urlPath, artifact) + }} + > + + + ) : null} + { + openDeleteArtifactDialog( + trial.study_id, + trial.trial_id, + artifact + ) + }} + > + + + + + + + + ) + })} + + + {renderDeleteArtifactDialog()} + {renderThreejsArtifactModal()} + + ) +} + +const TrialArtifactUploader: FC<{ + trial: Trial + width: string + height: string +}> = ({ trial, width, height }) => { + const theme = useTheme() + const action = actionCreator() + const [dragOver, setDragOver] = useState(false) + + if (trial.state !== "Running" && trial.state !== "Waiting") { + return null + } + const inputRef = useRef(null) + const handleClick: MouseEventHandler = () => { + if (!inputRef || !inputRef.current) { + return + } + inputRef.current.click() + } + const handleOnChange: ChangeEventHandler = (e) => { + const files = e.target.files + if (files === null) { + return + } + action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) + } + const handleDrop: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + const files = e.dataTransfer.files + setDragOver(false) + for (let i = 0; i < files.length; i++) { + action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) + } + } + const handleDragOver: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(true) + } + const handleDragLeave: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(false) + } + return ( + + + + + + Upload a New File + + Drag your file here or click to browse. + + + + + ) +} diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index bb70c119..6922003a 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -1,13 +1,4 @@ -import React, { - ChangeEventHandler, - DragEventHandler, - FC, - MouseEventHandler, - ReactNode, - useMemo, - useRef, - useState, -} from "react" +import React, { FC, ReactNode, useMemo } from "react" import { Typography, Box, @@ -16,13 +7,7 @@ import { IconButton, Menu, MenuItem, - Card, - CardContent, - CardMedia, - CardActionArea, - Modal, } from "@mui/material" -import { SxProps } from "@mui/system" import Chip from "@mui/material/Chip" import Divider from "@mui/material/Divider" import List from "@mui/material/List" @@ -33,11 +18,6 @@ import ListSubheader from "@mui/material/ListSubheader" import FilterListIcon from "@mui/icons-material/FilterList" import CheckBoxOutlineBlankIcon from "@mui/icons-material/CheckBoxOutlineBlank" import CheckBoxIcon from "@mui/icons-material/CheckBox" -import UploadFileIcon from "@mui/icons-material/UploadFile" -import DownloadIcon from "@mui/icons-material/Download" -import DeleteIcon from "@mui/icons-material/Delete" -import FullscreenIcon from "@mui/icons-material/Fullscreen" -import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile" import StopCircleIcon from "@mui/icons-material/StopCircle" import { TrialNote } from "./Note" @@ -46,9 +26,8 @@ import ListItemIcon from "@mui/material/ListItemIcon" import { useRecoilValue } from "recoil" import { artifactIsAvailable } from "../state" import { actionCreator } from "../action" -import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" import { TrialFormWidgets } from "./TrialFormWidgets" -import { ThreejsArtifactViewer } from "./ThreejsArtifactViewer" +import { TrialArtifactCards } from "./TrialArtifactCards" const states: TrialState[] = [ "Complete", @@ -320,344 +299,11 @@ export const TrialListDetail: FC<{ value !== null ? renderInfo(key, value) : null )} - {artifactEnabled && } + {artifactEnabled && } ) } -export const TrialArtifactContent: FC<{ - trial: Trial - artifact: Artifact - width: string - height: string -}> = ({ trial, artifact, width, height }) => { - if (artifact.mimetype.startsWith("image")) { - return ( - - ) - } else if ( - artifact.filename.endsWith(".stl") || - artifact.filename.endsWith(".3dm") - ) { - return ( - - - - ) - } else if (artifact.mimetype.startsWith("audio")) { - return ( - - - - ) - } else { - return ( - - - - ) - } -} - -export const TrialArtifactActions: FC<{ - trial: Trial - artifact: Artifact - sx: SxProps -}> = ({ trial, artifact, sx }) => { - const [open3dModelViewer, setOpen3dModelViewer] = useState(false) - - if (artifact.mimetype.startsWith("image")) { - return null - } else if ( - artifact.filename.endsWith(".stl") || - artifact.filename.endsWith(".3dm") - ) { - return ( - <> - { - setOpen3dModelViewer(true) - }} - > - - - { - setOpen3dModelViewer(false) - }} - > - - - - - - ) - } - return null -} - -const TrialArtifact: FC<{ - trial: Trial - artifact: Artifact - width: string - height: string -}> = ({ trial, artifact, width, height }) => { - const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = - useDeleteArtifactDialog() - const theme = useTheme() - const is3dModel = - artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") - const canDelete = trial.state === "Running" || trial.state === "Waiting" - let actionsCount = 1 - if (canDelete) actionsCount += 1 - if (is3dModel) actionsCount += 1 - const actionsWidth = theme.spacing(actionsCount * 4) - - return ( - - - - - {artifact.filename} - - {is3dModel ? ( - - ) : null} - {canDelete ? ( - { - openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) - }} - > - - - ) : null} - - - - - {renderDeleteArtifactDialog()} - - ) -} - -const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { - const theme = useTheme() - const action = actionCreator() - const [dragOver, setDragOver] = useState(false) - - const width = "200px" - const height = "150px" - - const inputRef = useRef(null) - const handleClick: MouseEventHandler = () => { - if (!inputRef || !inputRef.current) { - return - } - inputRef.current.click() - } - const handleOnChange: ChangeEventHandler = (e) => { - const files = e.target.files - if (files === null) { - return - } - action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) - } - const handleDrop: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - const files = e.dataTransfer.files - setDragOver(false) - for (let i = 0; i < files.length; i++) { - action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) - } - } - const handleDragOver: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(true) - } - const handleDragLeave: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(false) - } - - return ( - <> - - Artifacts - - - {trial.artifacts.map((a) => ( - - ))} - {trial.state === "Running" || trial.state === "Waiting" ? ( - - - - - - Upload a New File - - Drag your file here or click to browse. - - - - - ) : null} - - - ) -} - const getTrialListLink = ( studyId: number, exclude: TrialState[], diff --git a/optuna_dashboard/ts/components/UserDefinedPlot.tsx b/optuna_dashboard/ts/components/UserDefinedPlot.tsx new file mode 100644 index 00000000..029c4b57 --- /dev/null +++ b/optuna_dashboard/ts/components/UserDefinedPlot.tsx @@ -0,0 +1,21 @@ +import * as plotly from "plotly.js-dist-min" +import React, { FC, useEffect } from "react" +import { Box } from "@mui/material" + +export const UserDefinedPlot: FC<{ + graphObject: PlotlyGraphObject +}> = ({ graphObject }) => { + const plotDomId = `user-defined-plot:${graphObject.id}` + + useEffect(() => { + try { + const parsed = JSON.parse(graphObject.graph_object) + plotly.react(plotDomId, parsed.data, parsed.layout) + } catch (e) { + // Avoid to crash the whole page when given invalid grpah objects. + console.error(e) + } + }, [graphObject]) + + return +} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 241ea310..66fc06ed 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -12,6 +12,7 @@ type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type TrialStateFinished = "Complete" | "Fail" | "Pruned" type StudyDirection = "maximize" | "minimize" | "not_set" +type PreferenceFeedbackMode = "ChooseWorst" type FeedbackComponentType = "Note" | "Artifact" type FloatDistribution = { @@ -182,6 +183,11 @@ type FormWidgets = widgets: UserAttrFormWidget[] } +type PlotlyGraphObject = { + id: string + graph_object: string +} + type StudyDetail = { id: number name: string @@ -200,6 +206,8 @@ type StudyDetail = { form_widgets?: FormWidgets feedback_component_type?: FeedbackComponentType feedback_artifact_key?: string + preference_history?: PreferenceHistory[] + plotly_graph_objects: PlotlyGraphObject[] } type StudyDetails = { @@ -209,3 +217,12 @@ type StudyDetails = { type StudyParamImportance = { [study_id: string]: ParamImportance[][] } + +type PreferenceHistory = { + id: string + preference_id: string + candidates: number[] + clicked: number + feedback_mode: PreferenceFeedbackMode + timestamp: Date +} diff --git a/pyproject.toml b/pyproject.toml index 0555f701..9b7ce731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ docs = [ test = [ "coverage", + "plotly", "pytest", "moto[s3]", ] diff --git a/python_tests/preferential/test_study.py b/python_tests/preferential/test_study.py index 3de57dae..7fc1aaba 100644 --- a/python_tests/preferential/test_study.py +++ b/python_tests/preferential/test_study.py @@ -40,7 +40,6 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli for _ in range(2): trial = study.ask() trial.suggest_float("x", 0, 1) - study.mark_comparison_ready(trial) better, worse = study.trials study.report_preference(better, worse) assert len(study.preferences) == 1 @@ -152,7 +151,6 @@ def test_copy_study() -> None: for _ in range(3): trial = from_study.ask() trial.suggest_float("x", 0, 1) - from_study.mark_comparison_ready(trial) from_study.report_preference(from_study.trials[0], from_study.trials[1]) from_study.report_preference(from_study.trials[1], from_study.trials[2]) @@ -243,7 +241,6 @@ def test_get_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: for _ in range(5): trial = study.ask() trial.suggest_int("x", 1, 5) - study.mark_comparison_ready(trial) with patch("copy.deepcopy", wraps=copy.deepcopy) as mock_object: trials0 = study.get_trials(deepcopy=False) @@ -266,8 +263,7 @@ def test_get_trials_state_option(storage_supplier: Callable[[], StorageSupplier] with storage_supplier() as storage: study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() better, worse = study.trials[:2] study.report_preference(better, worse) diff --git a/python_tests/preferential/test_system_attrs.py b/python_tests/preferential/test_system_attrs.py index 10448d48..34f93200 100644 --- a/python_tests/preferential/test_system_attrs.py +++ b/python_tests/preferential/test_system_attrs.py @@ -18,12 +18,13 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli study.ask() study_id = study._study_id - assert len(get_preferences(study_id, storage)) == 0 + + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 0 better, worse = study.trials[0], study.trials[1] report_preferences(study_id, storage, [(better.number, worse.number)]) - assert len(get_preferences(study_id, storage)) == 1 + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 1 - actual_better, actual_worse = get_preferences(study_id, storage)[0] + actual_better, actual_worse = get_preferences(storage.get_study_system_attrs(study_id))[0] assert actual_better == better.number assert actual_worse == worse.number diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 7633cd8f..29e4da9f 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -105,10 +105,11 @@ class APITestCase(TestCase): storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() study.report_preference(study.trials[0], study.trials[1]) + assert len(study.best_trials) == 1 + app = create_app(storage) study_id = study._study._study_id status, _, body = send_request( @@ -120,16 +121,14 @@ class APITestCase(TestCase): self.assertEqual(status, 200) best_trials = json.loads(body)["best_trials"] - assert len(best_trials) == 2 + assert len(best_trials) == 1 assert best_trials[0]["number"] == 0 - assert best_trials[1]["number"] == 2 def test_report_preference(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id @@ -137,7 +136,13 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference", "POST", - body=json.dumps({"best_trials": [0, 2], "worst_trials": [1]}), + body=json.dumps( + { + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), content_type="application/json", ) self.assertEqual(status, 204) @@ -152,13 +157,35 @@ class APITestCase(TestCase): assert better.number == 2 assert worse.number == 1 + def test_report_preference_when_typo_mode(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference", + "POST", + body=json.dumps( + { + "mode": "ChoseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 400) + def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(storage=storage, n_generate=3) register_output_component(study, "Note") for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id @@ -220,23 +247,23 @@ class APITestCase(TestCase): trials: list[optuna.Trial] = [] for _ in range(3): trial = study.ask() - study.mark_comparison_ready(trial) trials.append(trial) + study.report_preference(study.trials[0], study.trials[1]) + study.report_preference(study.trials[2], study.trials[1]) app = create_app(storage) study_id = study._study._study_id status, _, _ = send_request( app, - f"/api/studies/{study_id}/{trials[1]._trial_id}/skip", + f"/api/studies/{study_id}/{trials[0]._trial_id}/skip", "POST", content_type="application/json", ) self.assertEqual(status, 204) best_trials = study.best_trials - assert len(best_trials) == 2 - assert best_trials[0].number == 0 - assert best_trials[1].number == 2 + assert len(best_trials) == 1 + assert best_trials[0].number == 2 def test_create_study(self) -> None: for name, directions, expected_status in [ diff --git a/python_tests/test_custom_plot_data.py b/python_tests/test_custom_plot_data.py new file mode 100644 index 00000000..3dcfc856 --- /dev/null +++ b/python_tests/test_custom_plot_data.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import optuna +from optuna_dashboard import _custom_plot_data as custom_plot_data +from optuna_dashboard import save_plotly_graph_object +import pytest + + +def get_dummy_study() -> optuna.Study: + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + optuna.logging.set_verbosity(optuna.logging.ERROR) + study.optimize(objective, n_trials=100) + return study + + +def test_save_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 2 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +def test_update_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object( + dummy_study, plot_data, graph_object_id=graph_object_id + ) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +@pytest.mark.parametrize( + "name", + [ + "0", + "a", + "a1-:_.", + ], +) +def test_is_valid_graph_object_id(name: str) -> None: + assert custom_plot_data.is_valid_graph_object_id(name) + + +@pytest.mark.parametrize( + "name", + [ + "a,", + "a b", + "aあいうえお", + ], +) +def test_is_invalid_graph_object_id(name: str) -> None: + assert not custom_plot_data.is_valid_graph_object_id(name) diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py new file mode 100644 index 00000000..51c9b0f8 --- /dev/null +++ b/python_tests/test_preferential_history.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import Callable + +from optuna_dashboard._preferential_history import NewHistory +from optuna_dashboard._preferential_history import report_history +from optuna_dashboard._serializer import serialize_preference_history +from optuna_dashboard.preferential import create_study +from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE + +from .storage_supplier import parametrize_storages +from .storage_supplier import StorageSupplier + + +@parametrize_storages +def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage, n_generate=5) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) + + study_id = study._study._study_id + + report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 1, 2], + clicked=1, + ), + ) + report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 2, 3, 4], + clicked=0, + ), + ) + history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + sys_attrs = storage.get_study_system_attrs(study_id) + assert len(history) == 2 + assert history[0]["candidates"] == [0, 1, 2] + assert history[0]["clicked"] == 1 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]] + assert len(preferences) == 2 + for i, (best, worst) in enumerate([(0, 1), (2, 1)]): + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst + assert history[1]["candidates"] == [0, 2, 3, 4] + assert history[1]["clicked"] == 0 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]] + assert len(preferences) == 3 + for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index a90e0de7..72db7b26 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -29,7 +29,7 @@ def test_get_study_detail_is_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert study_detail["is_preferential"] @@ -40,7 +40,7 @@ def test_get_study_detail_is_not_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert not study_detail["is_preferential"] From ded56df9fc54abe0cf34047f7e891a9ded690565 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 15:35:04 +0900 Subject: [PATCH 09/38] split component --- .../ts/components/PreferentialTrials.tsx | 377 +++++++++--------- 1 file changed, 196 insertions(+), 181 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index b7805259..b1ba6b2c 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -163,14 +163,12 @@ const SettingsPage: FC<{ ) } -const FeedbackContent: FC<{ +const OutputContent: FC<{ trial: Trial artifact?: Artifact componentId: FeedbackComponentType - width: string - minHeight: string urlPath: string -}> = ({ trial, artifact, componentId, width, minHeight, urlPath }) => { +}> = ({ trial, artifact, componentId, urlPath }) => { if (componentId === "Note") { return } @@ -186,6 +184,191 @@ const FeedbackContent: FC<{ return null } +const PreferentialTrial: FC<{ + trial?: Trial + studyDetail: StudyDetail + candidates: number[] + hideTrial: () => void + openDetailTrial: () => void + openThreejsArtifactModal: (urlPath: string, artifact: Artifact) => void +}> = ({ + trial, + studyDetail, + candidates, + hideTrial, + openDetailTrial, + openThreejsArtifactModal, +}) => { + const theme = useTheme() + const action = actionCreator() + const [buttonHover, setButtonHover] = useState(false) + const trialWidth = 400 + const trialHeight = 300 + const componentId = studyDetail.feedback_component_type ?? "Note" + const artifactKey = studyDetail.feedback_artifact_key + const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value + const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) + const urlPath = `/artifacts/${studyDetail.id}/${trial?.trial_id}/${artifact?.artifact_id}` + const is3dModel = + componentId === "Artifact" && + artifact !== undefined && + isThreejsArtifact(artifact) + + if (trial === undefined) { + return ( + + ) + } + + const onFeedback = () => { + hideTrial() + action.updatePreference(trial.study_id, candidates, trial.number) + } + + return ( + + + + Trial {trial.number} + {componentId === "Artifact" && artifact !== undefined ? ( + + {`(${artifact.filename})`} + + ) : null} + + {is3dModel ? ( + { + openThreejsArtifactModal(urlPath, artifact) + }} + > + + + ) : null} + { + hideTrial() + action.skipPreferentialTrial(trial.study_id, trial.trial_id) + }} + aria-label="skip trial" + > + + + + + + + { + if (e.shiftKey) onFeedback() + }} + sx={{ + position: "relative", + padding: theme.spacing(2), + overflow: "hidden", + minHeight: theme.spacing(20), + }} + > + + + + + + + ) +} + type DisplayTrials = { numbers: number[] last_number: number @@ -195,7 +378,6 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail, }) => { const theme = useTheme() - const action = actionCreator() const [openThreejsArtifactModal, renderThreejsArtifactModal] = useThreejsArtifactModal() const runningTrials = @@ -208,10 +390,6 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ }) const [settingShown, setSettingShown] = useState(false) const [detailTrial, setDetailTrial] = useState(null) - const [buttonHover, setButtonHover] = useState(null) - - const trialWidth = 400 - const trialHeight = 300 if (studyDetail === null || !studyDetail.is_preferential) { return null @@ -285,179 +463,16 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {displayTrials.numbers.map((t, index) => { const trial = activeTrials.find((trial) => trial.number === t) const candidates = displayTrials.numbers.filter((n) => n !== -1) - const componentId = studyDetail.feedback_component_type ?? "Note" - const artifactKey = studyDetail.feedback_artifact_key - const artifactId = trial?.user_attrs.find( - (a) => a.key === artifactKey - )?.value - const artifact = trial?.artifacts.find( - (a) => a.artifact_id === artifactId - ) - const urlPath = `/artifacts/${studyDetail.id}/${trial?.trial_id}/${artifact?.artifact_id}` - - if (trial == undefined) { - return ( - - ) - } - - const is3dModel = - componentId === "Artifact" && - artifact !== undefined && - isThreejsArtifact(artifact) - const onFeedback = () => { - hideTrial(trial.number) - action.updatePreference(trial.study_id, candidates, trial.number) - } - return ( - - - - Trial {trial.number} - {componentId === "Artifact" && artifact !== undefined ? ( - - {`(${artifact.filename})`} - - ) : null} - - {is3dModel ? ( - { - openThreejsArtifactModal(urlPath, artifact) - }} - > - - - ) : null} - { - hideTrial(trial.number) - action.skipPreferentialTrial(trial.study_id, trial.trial_id) - }} - aria-label="skip trial" - > - - - setDetailTrial(trial.number)} - aria-label="show detail" - > - - - - { - if (e.shiftKey) onFeedback() - }} - sx={{ - position: "relative", - padding: theme.spacing(2), - overflow: "hidden", - minHeight: theme.spacing(20), - }} - > - - - - - - + hideTrial(t)} + openDetailTrial={() => setDetailTrial(t)} + openThreejsArtifactModal={openThreejsArtifactModal} + /> ) })} From d218818a72f1490a2e781a9ed5e33ba27b7d43e6 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 15:42:52 +0900 Subject: [PATCH 10/38] fix by lint --- python_tests/test_api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 29e4da9f..e22d104c 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -212,11 +212,10 @@ class APITestCase(TestCase): def test_change_component_type_only(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(storage=storage, n_generate=3) register_output_component(study, "Artifact", "audio") for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id From 4a7c26fc9042f7d37360b4396db0984bc291128d Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 17:43:44 +0900 Subject: [PATCH 11/38] follow output component in history --- optuna_dashboard/ts/apiClient.ts | 7 ++++-- .../ts/components/PreferenceHistory.tsx | 23 +++++++++++++++++-- .../ts/components/PreferentialTrials.tsx | 17 +++++++++++--- optuna_dashboard/ts/types/index.d.ts | 2 +- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 6ae34403..53a23430 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -94,6 +94,8 @@ interface StudyDetailResponse { form_widgets?: FormWidgets preference_history?: PreferenceHistoryResponce[] plotly_graph_objects: PlotlyGraphObject[] + feedback_component_type?: FeedbackComponentType + feedback_artifact_key?: string } export const getStudyDetailAPI = ( @@ -129,8 +131,9 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, - feedback_component_type: res.data - .feedback_component_type as FeedbackComponentType, + feedback_component_type: res.data.feedback_component_type + ? (res.data.feedback_component_type as FeedbackComponentType) + : "Note", feedback_artifact_key: res.data.feedback_artifact_key, preference_history: res.data.preference_history?.map( convertPreferenceHistory diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 3bc5a750..c290d7c6 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -14,8 +14,9 @@ import Modal from "@mui/material/Modal" import { red } from "@mui/material/colors" import { TrialListDetail } from "./TrialList" -import { MarkdownRenderer } from "./Note" +import { OutputContent, getArtifactUrlPath } from "./PreferentialTrials" import { formatDate } from "../dateUtil" +import { useStudyDetailValue } from "../state" type TrialType = "worst" | "none" @@ -26,8 +27,21 @@ const CandidateTrial: FC<{ const theme = useTheme() const trialWidth = 300 const trialHeight = 300 + const studyDetail = useStudyDetailValue(trial.study_id) const [detailShown, setDetailShown] = useState(false) + if (studyDetail === null) { + return null + } + const componentId = studyDetail.feedback_component_type + const artifactKey = studyDetail.feedback_artifact_key + const artifactId = trial.user_attrs.find((a) => a.key === artifactKey)?.value + const artifact = trial.artifacts.find((a) => a.artifact_id === artifactId) + const urlPath = + artifactId !== undefined + ? getArtifactUrlPath(trial.study_id, trial.trial_id, artifactId) + : "" + const cardComponentSx = { padding: 0, position: "relative", @@ -76,7 +90,12 @@ const CandidateTrial: FC<{ padding: theme.spacing(2), }} > - + {type === "worst" ? ( diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index b1ba6b2c..9e93a294 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -163,7 +163,7 @@ const SettingsPage: FC<{ ) } -const OutputContent: FC<{ +export const OutputContent: FC<{ trial: Trial artifact?: Artifact componentId: FeedbackComponentType @@ -184,6 +184,14 @@ const OutputContent: FC<{ return null } +export const getArtifactUrlPath = ( + studyId: number, + trialId: number, + artifactId: string +) => { + return `/artifacts/${studyId}/${trialId}/${artifactId}` +} + const PreferentialTrial: FC<{ trial?: Trial studyDetail: StudyDetail @@ -204,11 +212,14 @@ const PreferentialTrial: FC<{ const [buttonHover, setButtonHover] = useState(false) const trialWidth = 400 const trialHeight = 300 - const componentId = studyDetail.feedback_component_type ?? "Note" + const componentId = studyDetail.feedback_component_type const artifactKey = studyDetail.feedback_artifact_key const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) - const urlPath = `/artifacts/${studyDetail.id}/${trial?.trial_id}/${artifact?.artifact_id}` + const urlPath = + trial !== undefined && artifactId !== undefined + ? getArtifactUrlPath(studyDetail.id, trial?.trial_id, artifactId) + : "" const is3dModel = componentId === "Artifact" && artifact !== undefined && diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 66fc06ed..88ece9a5 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -204,7 +204,7 @@ type StudyDetail = { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets - feedback_component_type?: FeedbackComponentType + feedback_component_type: FeedbackComponentType feedback_artifact_key?: string preference_history?: PreferenceHistory[] plotly_graph_objects: PlotlyGraphObject[] From ee9ebafcf46a6c2cb2998df163e64f8ef4d62d36 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 17:59:34 +0900 Subject: [PATCH 12/38] follow output component in graph --- .../ts/components/PreferentialGraph.tsx | 29 ++++++++++++++----- .../ts/components/PreferentialTrials.tsx | 6 ++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialGraph.tsx b/optuna_dashboard/ts/components/PreferentialGraph.tsx index 48364598..a52c8571 100644 --- a/optuna_dashboard/ts/components/PreferentialGraph.tsx +++ b/optuna_dashboard/ts/components/PreferentialGraph.tsx @@ -1,4 +1,4 @@ -import React, { FC, useState, useCallback, useMemo, useEffect } from "react" +import React, { FC, useState, useCallback, useEffect } from "react" import { Card, CardContent, @@ -7,7 +7,6 @@ import { Box, Chip, } from "@mui/material" -import { MarkdownRenderer } from "./Note" import ReactFlow, { Node, NodeProps, @@ -24,6 +23,9 @@ import "reactflow/dist/style.css" import ELK from "elkjs/lib/elk.bundled.js" import { ElkNode } from "elkjs/lib/elk-api.js" +import { useStudyDetailValue } from "../state" +import { OutputContent, getArtifactUrlPath } from "./PreferentialTrials" + const elk = new ELK() const nodeWidth = 400 const nodeHeight = 300 @@ -39,10 +41,16 @@ const GraphNode: FC> = ({ data, isConnectable }) => { if (trial === undefined) { return null } - const noteBody = trial.note.body - const noteFC = useMemo(() => { - return - }, [noteBody]) + const studyDetail = useStudyDetailValue(trial.study_id) + const componentId = studyDetail?.feedback_component_type + const artifactKey = studyDetail?.feedback_artifact_key + const artifactId = trial.user_attrs.find((a) => a.key === artifactKey)?.value + const artifact = trial.artifacts.find((a) => a.artifact_id === artifactId) + const urlPath = + artifactId !== undefined + ? getArtifactUrlPath(trial.study_id, trial.trial_id, artifactId) + : "" + return ( > = ({ data, isConnectable }) => { style={{ background: "#555" }} isConnectable={isConnectable} /> - {noteFC} + + + = ({ trial, artifact, componentId, urlPath }) => { - if (componentId === "Note") { + if (componentId === undefined || componentId === "Note") { return } if (componentId === "Artifact") { @@ -188,7 +188,7 @@ export const getArtifactUrlPath = ( studyId: number, trialId: number, artifactId: string -) => { +): string => { return `/artifacts/${studyId}/${trialId}/${artifactId}` } From b4a4c2dae0bdac885634836ad8ba9ee871f35ab3 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 15:28:24 +0900 Subject: [PATCH 13/38] split api part of preference feedback component --- optuna_dashboard/_app.py | 23 ++++++++++ optuna_dashboard/_preference_setting.py | 60 ++++++++++++++++++++++++ optuna_dashboard/_serializer.py | 6 +++ python_tests/test_api.py | 61 +++++++++++++++++++++++++ python_tests/test_preference_setting.py | 18 ++++++++ 5 files changed, 168 insertions(+) create mode 100644 optuna_dashboard/_preference_setting.py create mode 100644 python_tests/test_preference_setting.py diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index c32c2061..fad4b74c 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,6 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preference_setting import _register_output_component from ._preferential_history import NewHistory from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route @@ -306,6 +307,28 @@ def create_app( response.status = 204 return {} + @app.post("/api/studies//component") + @json_api_view + def post_component(study_id: int) -> dict[str, Any]: + try: + component_type = request.json.get("component_type", "") + artifact_key = request.json.get("artifact_key", None) + except ValueError: + response.status = 400 + return {"reason": "invalid request."} + if component_type not in ["Note", "Artifact"]: + response.status = 400 + return {"reason": "component_type must be either 'Note' or 'Artifact'."} + + _register_output_component( + study_id=study_id, + storage=storage, + component_type=component_type, + artifact_key=artifact_key, + ) + response.status = 204 + return {} + @app.post("/api/trials//tell") @json_api_view def tell_trial(trial_id: int) -> dict[str, Any]: diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py new file mode 100644 index 00000000..93b1a34c --- /dev/null +++ b/optuna_dashboard/_preference_setting.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from optuna.storages import BaseStorage + +from .preferential._study import PreferentialStudy + + +if TYPE_CHECKING: + from typing import Literal + + OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] + +_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE = "preference:component_type" +_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY = "preference:component_artifact_key" + + +def _register_output_component( + study_id: int, + storage: BaseStorage, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str | None = None, +) -> None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, + value=component_type, + ) + if artifact_key is not None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, + value=artifact_key, + ) + + +def register_output_component( + study: PreferentialStudy, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str = "", +) -> None: + """Register output component to the study. + + Args: + study: + The study to register the output component. + component_type: + The type of the output component. + artifact_key: + When the component_type is "Artifact", + this argument is used as the attribute key of the artifact. + Each trial displays the artifact whose id is the value of the attribute. + """ + _register_output_component( + study_id=study._study._study_id, + storage=study._study._storage, + component_type=component_type, + artifact_key=artifact_key, + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index e3f77649..511e7365 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,6 +15,8 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -162,6 +164,10 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: + serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] + if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: + serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index c551e3f2..e22d104c 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,6 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study +from optuna_dashboard._preference_setting import register_output_component from optuna_dashboard.preferential import create_study from .wsgi_client import send_request @@ -179,6 +180,66 @@ class APITestCase(TestCase): ) self.assertEqual(status, 400) + def test_change_component(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + register_output_component(study, "Note") + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Artifact", "artifact_key": "image"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Artifact" + assert study_detail["feedback_artifact_key"] == "image" + + def test_change_component_type_only(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + register_output_component(study, "Artifact", "audio") + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Note"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Note" + assert study_detail["feedback_artifact_key"] == "audio" + def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py new file mode 100644 index 00000000..033a9092 --- /dev/null +++ b/python_tests/test_preference_setting.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from unittest import TestCase + +import optuna +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard.preferential._study import PreferentialStudy + + +class FeedbackSettingTestCase(TestCase): + def test_widget_to_dict_from_dict(self) -> None: + study = PreferentialStudy(optuna.create_study()) + register_output_component(study, "Artifact", "image_key") + system_attrs = study._study.system_attrs + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, "") == "Artifact" + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, "") == "image_key" From 5fcc64cff3e44bb34fb7ab9f2296fd7a8b4ff4e4 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 16:00:46 +0900 Subject: [PATCH 14/38] fix by review --- optuna_dashboard/_app.py | 10 +++--- optuna_dashboard/_preference_setting.py | 24 ++++++-------- optuna_dashboard/_serializer.py | 10 +++--- python_tests/test_api.py | 44 ++++--------------------- python_tests/test_preference_setting.py | 14 ++++---- 5 files changed, 34 insertions(+), 68 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index fad4b74c..2ad7874e 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,7 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preference_setting import _register_output_component +from ._preference_setting import _register_preference_feedback_component_type from ._preferential_history import NewHistory from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route @@ -307,11 +307,11 @@ def create_app( response.status = 204 return {} - @app.post("/api/studies//component") + @app.put("/api/studies//preference_feedback_component_type") @json_api_view - def post_component(study_id: int) -> dict[str, Any]: + def put_component(study_id: int) -> dict[str, Any]: try: - component_type = request.json.get("component_type", "") + component_type = request.json.get("type", "") artifact_key = request.json.get("artifact_key", None) except ValueError: response.status = 400 @@ -320,7 +320,7 @@ def create_app( response.status = 400 return {"reason": "component_type must be either 'Note' or 'Artifact'."} - _register_output_component( + _register_preference_feedback_component_type( study_id=study_id, storage=storage, component_type=component_type, diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 93b1a34c..73e9b489 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -12,30 +12,26 @@ if TYPE_CHECKING: OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] -_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE = "preference:component_type" -_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY = "preference:component_artifact_key" +_SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" -def _register_output_component( +def _register_preference_feedback_component_type( study_id: int, storage: BaseStorage, component_type: OUTPUT_COMPONENT_TYPE, - artifact_key: str | None = None, + artifact_key: str = "", ) -> None: storage.set_study_system_attr( study_id=study_id, - key=_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, - value=component_type, + key=_SYSTEM_ATTR_FEEDBACK_COMPONENT, + value={ + "type": component_type, + "artifact_key": artifact_key, + } ) - if artifact_key is not None: - storage.set_study_system_attr( - study_id=study_id, - key=_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, - value=artifact_key, - ) -def register_output_component( +def register_preference_feedback_component_type( study: PreferentialStudy, component_type: OUTPUT_COMPONENT_TYPE, artifact_key: str = "", @@ -52,7 +48,7 @@ def register_output_component( this argument is used as the attribute key of the artifact. Each trial displays the artifact whose id is the value of the attribute. """ - _register_output_component( + _register_preference_feedback_component_type( study_id=study._study._study_id, storage=study._study._storage, component_type=component_type, diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 511e7365..98db5ffb 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,8 +15,7 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names -from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY -from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -164,10 +163,9 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets - if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: - serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] - if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: - serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] + if serialized["is_preferential"]: + serialized["feedback_component_type"] = system_attrs.get( + _SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index e22d104c..1908c188 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,7 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard._preference_setting import register_preference_feedback_component_type from optuna_dashboard.preferential import create_study from .wsgi_client import send_request @@ -183,7 +183,7 @@ class APITestCase(TestCase): def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) - register_output_component(study, "Note") + register_preference_feedback_component_type(study, "Note") for _ in range(3): study.ask() @@ -191,9 +191,9 @@ class APITestCase(TestCase): study_id = study._study._study_id status, _, _ = send_request( app, - f"/api/studies/{study_id}/component", - "POST", - body=json.dumps({"component_type": "Artifact", "artifact_key": "image"}), + f"/api/studies/{study_id}/preference_feedback_component_type", + "PUT", + body=json.dumps({"type": "Artifact", "artifact_key": "image"}), content_type="application/json", ) self.assertEqual(status, 204) @@ -207,38 +207,8 @@ class APITestCase(TestCase): self.assertEqual(status, 200) study_detail = json.loads(body) - assert study_detail["feedback_component_type"] == "Artifact" - assert study_detail["feedback_artifact_key"] == "image" - - def test_change_component_type_only(self) -> None: - storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage, n_generate=3) - register_output_component(study, "Artifact", "audio") - for _ in range(3): - study.ask() - - app = create_app(storage) - study_id = study._study._study_id - status, _, _ = send_request( - app, - f"/api/studies/{study_id}/component", - "POST", - body=json.dumps({"component_type": "Note"}), - content_type="application/json", - ) - self.assertEqual(status, 204) - - status, _, body = send_request( - app, - f"/api/studies/{study_id}", - "GET", - content_type="application/json", - ) - self.assertEqual(status, 200) - - study_detail = json.loads(body) - assert study_detail["feedback_component_type"] == "Note" - assert study_detail["feedback_artifact_key"] == "audio" + assert study_detail["feedback_component_type"]["type"] == "Artifact" + assert study_detail["feedback_component_type"]["artifact_key"] == "image" def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index 033a9092..8a601af7 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -3,16 +3,18 @@ from __future__ import annotations from unittest import TestCase import optuna -from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY -from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE -from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT +from optuna_dashboard._preference_setting import register_preference_feedback_component_type from optuna_dashboard.preferential._study import PreferentialStudy class FeedbackSettingTestCase(TestCase): def test_widget_to_dict_from_dict(self) -> None: study = PreferentialStudy(optuna.create_study()) - register_output_component(study, "Artifact", "image_key") + register_preference_feedback_component_type(study, "Artifact", "image_key") system_attrs = study._study.system_attrs - assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, "") == "Artifact" - assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, "") == "image_key" + feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) + assert "type" in feedback_type + assert feedback_type["type"] == "Artifact" + assert "artifact_key" in feedback_type + assert feedback_type["artifact_key"] == "image_key" From fffa586da3f197b8ab3f39243d428a75c0143075 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 16:04:40 +0900 Subject: [PATCH 15/38] fix by review --- optuna_dashboard/_preference_setting.py | 2 +- optuna_dashboard/_serializer.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 73e9b489..d9d7c1c2 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -27,7 +27,7 @@ def _register_preference_feedback_component_type( value={ "type": component_type, "artifact_key": artifact_key, - } + }, ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 98db5ffb..f9aeabec 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -165,7 +165,8 @@ def serialize_study_detail( serialized["form_widgets"] = form_widgets if serialized["is_preferential"]: serialized["feedback_component_type"] = system_attrs.get( - _SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) + _SYSTEM_ATTR_FEEDBACK_COMPONENT, {} + ) if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) From 702b35622e558b00d2ba42b546d8215d750553fc Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 17:54:42 +0900 Subject: [PATCH 16/38] fix by review --- optuna_dashboard/_app.py | 2 +- optuna_dashboard/_preference_setting.py | 20 +++++++++++++------- python_tests/test_api.py | 6 +++--- python_tests/test_preference_setting.py | 4 ++-- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 2ad7874e..d7ddf72b 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -316,7 +316,7 @@ def create_app( except ValueError: response.status = 400 return {"reason": "invalid request."} - if component_type not in ["Note", "Artifact"]: + if component_type not in ["note", "artifact"]: response.status = 400 return {"reason": "component_type must be either 'Note' or 'Artifact'."} diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index d9d7c1c2..2310aa03 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING +from typing import Any from optuna.storages import BaseStorage @@ -10,7 +11,7 @@ from .preferential._study import PreferentialStudy if TYPE_CHECKING: from typing import Literal - OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] + OUTPUT_COMPONENT_TYPE = Literal["note", "artifact"] _SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" @@ -19,22 +20,22 @@ def _register_preference_feedback_component_type( study_id: int, storage: BaseStorage, component_type: OUTPUT_COMPONENT_TYPE, - artifact_key: str = "", + artifact_key: str | None = None, ) -> None: + value: dict[str, Any] = {"type": component_type} + if artifact_key is not None: + value["artifact_key"] = artifact_key storage.set_study_system_attr( study_id=study_id, key=_SYSTEM_ATTR_FEEDBACK_COMPONENT, - value={ - "type": component_type, - "artifact_key": artifact_key, - }, + value=value, ) def register_preference_feedback_component_type( study: PreferentialStudy, component_type: OUTPUT_COMPONENT_TYPE, - artifact_key: str = "", + artifact_key: str | None = None, ) -> None: """Register output component to the study. @@ -48,6 +49,11 @@ def register_preference_feedback_component_type( this argument is used as the attribute key of the artifact. Each trial displays the artifact whose id is the value of the attribute. """ + if component_type == "artifact": + assert ( + artifact_key is not None + ), "artifact_key must be specified when component_type is Artifact" + _register_preference_feedback_component_type( study_id=study._study._study_id, storage=study._study._storage, diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 1908c188..c478dc26 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -183,7 +183,7 @@ class APITestCase(TestCase): def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) - register_preference_feedback_component_type(study, "Note") + register_preference_feedback_component_type(study, "note") for _ in range(3): study.ask() @@ -193,7 +193,7 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference_feedback_component_type", "PUT", - body=json.dumps({"type": "Artifact", "artifact_key": "image"}), + body=json.dumps({"type": "artifact", "artifact_key": "image"}), content_type="application/json", ) self.assertEqual(status, 204) @@ -207,7 +207,7 @@ class APITestCase(TestCase): self.assertEqual(status, 200) study_detail = json.loads(body) - assert study_detail["feedback_component_type"]["type"] == "Artifact" + assert study_detail["feedback_component_type"]["type"] == "artifact" assert study_detail["feedback_component_type"]["artifact_key"] == "image" def test_skip_trial(self) -> None: diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index 8a601af7..bcd53e42 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -11,10 +11,10 @@ from optuna_dashboard.preferential._study import PreferentialStudy class FeedbackSettingTestCase(TestCase): def test_widget_to_dict_from_dict(self) -> None: study = PreferentialStudy(optuna.create_study()) - register_preference_feedback_component_type(study, "Artifact", "image_key") + register_preference_feedback_component_type(study, "artifact", "image_key") system_attrs = study._study.system_attrs feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) assert "type" in feedback_type - assert feedback_type["type"] == "Artifact" + assert feedback_type["type"] == "artifact" assert "artifact_key" in feedback_type assert feedback_type["artifact_key"] == "image_key" From 8e23d30d182f757def64b0c907d7d88a6517e45e Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 18:18:19 +0900 Subject: [PATCH 17/38] fix by format --- optuna_dashboard/_preference_setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 2310aa03..acc64ccd 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING from typing import Any +from typing import TYPE_CHECKING from optuna.storages import BaseStorage From 32a4f2d4e1c267dba695c9e4828c376e32e5eb95 Mon Sep 17 00:00:00 2001 From: contramundum53 Date: Thu, 14 Sep 2023 10:36:20 +0900 Subject: [PATCH 18/38] Update test_api.py --- python_tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 2e46be43..d5de330d 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,8 +8,8 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preference_setting import register_preference_feedback_component_type +from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history from optuna_dashboard._serializer import serialize_preference_history From 88e3d511d5ff607d44b780205849b92efa6b6792 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 12:03:17 +0900 Subject: [PATCH 19/38] fix by review --- optuna_dashboard/_app.py | 4 +++- optuna_dashboard/_serializer.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 1baf76fe..4c22bdbf 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -312,7 +312,7 @@ def create_app( @app.put("/api/studies//preference_feedback_component_type") @json_api_view - def put_component(study_id: int) -> dict[str, Any]: + def put_preference_feedback_component_type(study_id: int) -> dict[str, Any]: try: component_type = request.json.get("type", "") artifact_key = request.json.get("artifact_key", None) @@ -329,6 +329,8 @@ def create_app( component_type=component_type, artifact_key=artifact_key, ) + response.status = 204 + return {} @app.delete("/api/studies//preference/") @json_api_view diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index ea22644f..b229b738 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -168,7 +168,6 @@ def serialize_study_detail( serialized["feedback_component_type"] = system_attrs.get( _SYSTEM_ATTR_FEEDBACK_COMPONENT, {} ) - if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) serialized["plotly_graph_objects"] = [ From 4e031585b2d90b290d6bcd1dca5ccd50547a448a Mon Sep 17 00:00:00 2001 From: Contramundum Date: Thu, 14 Sep 2023 13:40:19 +0900 Subject: [PATCH 20/38] Support all-categorical cases --- optuna_dashboard/preferential/samplers/gp.py | 42 +++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 8349b1a4..349da03a 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -3,6 +3,7 @@ from __future__ import annotations import math from typing import Any from typing import Callable +from typing import cast import botorch.acquisition.analytic import botorch.models.model @@ -16,6 +17,8 @@ import optuna import optuna._transform import torch from torch import Tensor +from optuna.distributions import CategoricalDistribution +import itertools from .._system_attrs import get_preferences @@ -310,7 +313,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): search_space: dict[str, optuna.distributions.BaseDistribution], ) -> dict[str, Any]: preferences = get_preferences(study.system_attrs) - if len(preferences) == 0: + if len(preferences) == 0 or len(search_space) == 0: return {} trials = study.get_trials(deepcopy=False) @@ -355,16 +358,33 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): best_f=torch.max(sampled_gp.posterior(params[:, None, :]).mean), ) - # TODO: Make it possible to apply it on categorical variables - candidates, _ = botorch.optim.optimize_acqf( - acq_function=acqf, - bounds=torch.from_numpy(trans.bounds.T), - q=1, - num_restarts=10, - raw_samples=512, - options={"batch_limit": 5, "maxiter": 200}, - sequential=True, - ) + # TODO: Make it possible to apply it on mixed search space + if all(isinstance(dist, CategoricalDistribution) for dist in search_space.values()): + all_param_combinations = itertools.product( + *[ + [(name, choice) for choice in cast(CategoricalDistribution, dist).choices] + for name, dist in search_space.items() + ] + ) + choices = torch.tensor( + np.array([trans.transform(dict(params)) for params in all_param_combinations]), + dtype=torch.float64, + ) + candidates, _ = botorch.optim.optimize_acqf_discrete( + acq_function=acqf, + choices=choices, + q=1, + ) + else: + candidates, _ = botorch.optim.optimize_acqf( + acq_function=acqf, + bounds=torch.from_numpy(trans.bounds.T), + q=1, + num_restarts=10, + raw_samples=512, + options={"batch_limit": 5, "maxiter": 200}, + sequential=True, + ) next_x = trans.untransform(candidates[0].detach().numpy()) return next_x From b95f13e9fe9dc857bddf489666d8aea97374a8c1 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Thu, 14 Sep 2023 13:45:22 +0900 Subject: [PATCH 21/38] format --- optuna_dashboard/preferential/samplers/gp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 349da03a..d023e741 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -1,5 +1,6 @@ from __future__ import annotations +import itertools import math from typing import Any from typing import Callable @@ -15,10 +16,9 @@ from gpytorch.likelihoods.gaussian_likelihood import Prior import numpy as np import optuna import optuna._transform +from optuna.distributions import CategoricalDistribution import torch from torch import Tensor -from optuna.distributions import CategoricalDistribution -import itertools from .._system_attrs import get_preferences From e5e6d0766f69c69332d3c9d149e71f8638ade1c1 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 15:37:48 +0900 Subject: [PATCH 22/38] fix docstring --- optuna_dashboard/_preference_setting.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index acc64ccd..167fd178 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -37,17 +37,22 @@ def register_preference_feedback_component_type( component_type: OUTPUT_COMPONENT_TYPE, artifact_key: str | None = None, ) -> None: - """Register output component to the study. + """Register a preference feedback component to the study. + With this feature, you can change the component, displayed on the + human feedback pages. By default, the Markdown note (``component_type="note"``) + is displayed. If you specify ``component_type="artifact"``, the viewer for the + specified artifact file will be displayed. Args: study: - The study to register the output component. + The study to register the preference feedback component. component_type: - The type of the output component. - artifact_key: - When the component_type is "Artifact", - this argument is used as the attribute key of the artifact. - Each trial displays the artifact whose id is the value of the attribute. + The component type, displayed on the human feedback pages + (default: ``"note"``). + user_attr_artifact_key: + This option is required when the ``component_type`` is ``"artifact"``. + The user attribute, which is specified this field, must contain the + ``artifact``id you want to display on the human feedback page. """ if component_type == "artifact": assert ( From 9d7f2f9a6e141450f388747782ef9de2f228b141 Mon Sep 17 00:00:00 2001 From: moririn2528 <49509238+moririn2528@users.noreply.github.com> Date: Thu, 14 Sep 2023 16:28:43 +0900 Subject: [PATCH 23/38] Update optuna_dashboard/_preference_setting.py Co-authored-by: c-bata --- optuna_dashboard/_preference_setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 167fd178..d0f8c82c 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -32,7 +32,7 @@ def _register_preference_feedback_component_type( ) -def register_preference_feedback_component_type( +def register_preference_feedback_component( study: PreferentialStudy, component_type: OUTPUT_COMPONENT_TYPE, artifact_key: str | None = None, From 4570efe74db693276482aa946bfec4be7cec3064 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 16:39:44 +0900 Subject: [PATCH 24/38] fix by review --- docs/api.rst | 1 + optuna_dashboard/_app.py | 4 ++-- optuna_dashboard/_preference_setting.py | 4 ++-- python_tests/test_api.py | 4 ++-- python_tests/test_preference_setting.py | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index aadd2718..e66ae468 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,6 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy + optuna_dashboard._prefential_setting.register_preference_feedback_component Streamlit ----------------- diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 4c22bdbf..3abac3fc 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,7 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preference_setting import _register_preference_feedback_component_type +from ._preference_setting import _register_preference_feedback_component from ._preferential_history import NewHistory from ._preferential_history import PreferenceHistoryNotFound from ._preferential_history import remove_history @@ -323,7 +323,7 @@ def create_app( response.status = 400 return {"reason": "component_type must be either 'Note' or 'Artifact'."} - _register_preference_feedback_component_type( + _register_preference_feedback_component( study_id=study_id, storage=storage, component_type=component_type, diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index d0f8c82c..907fc2be 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: _SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" -def _register_preference_feedback_component_type( +def _register_preference_feedback_component( study_id: int, storage: BaseStorage, component_type: OUTPUT_COMPONENT_TYPE, @@ -59,7 +59,7 @@ def register_preference_feedback_component( artifact_key is not None ), "artifact_key must be specified when component_type is Artifact" - _register_preference_feedback_component_type( + _register_preference_feedback_component( study_id=study._study._study_id, storage=study._study._storage, component_type=component_type, diff --git a/python_tests/test_api.py b/python_tests/test_api.py index d5de330d..db79c0d8 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,7 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard._preference_setting import register_preference_feedback_component_type +from optuna_dashboard._preference_setting import register_preference_feedback_component from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history @@ -187,7 +187,7 @@ class APITestCase(TestCase): def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) - register_preference_feedback_component_type(study, "note") + register_preference_feedback_component(study, "note") for _ in range(3): study.ask() diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index bcd53e42..a0adcb39 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -4,14 +4,14 @@ from unittest import TestCase import optuna from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT -from optuna_dashboard._preference_setting import register_preference_feedback_component_type +from optuna_dashboard._preference_setting import register_preference_feedback_component from optuna_dashboard.preferential._study import PreferentialStudy class FeedbackSettingTestCase(TestCase): def test_widget_to_dict_from_dict(self) -> None: study = PreferentialStudy(optuna.create_study()) - register_preference_feedback_component_type(study, "artifact", "image_key") + register_preference_feedback_component(study, "artifact", "image_key") system_attrs = study._study.system_attrs feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) assert "type" in feedback_type From df1948a484f896aac4ef9e3576c36c7481a6ac13 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 16:42:58 +0900 Subject: [PATCH 25/38] minor fix --- docs/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index e66ae468..8da91536 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,7 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy - optuna_dashboard._prefential_setting.register_preference_feedback_component + optuna_dashboard._preference_setting.register_preference_feedback_component Streamlit ----------------- From 9f17603741a73770ebebec2d87356d2d1ef7ccda Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 17:05:12 +0900 Subject: [PATCH 26/38] minor fix --- docs/api.rst | 2 +- optuna_dashboard/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index 8da91536..18f8bc72 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,7 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy - optuna_dashboard._preference_setting.register_preference_feedback_component + optuna_dashboard.register_preference_feedback_component Streamlit ----------------- diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 5bb3f301..ea2a8dbe 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -14,6 +14,7 @@ from ._form_widget import TextInputWidget # noqa from ._named_objectives import set_objective_names # noqa from ._note import get_note # noqa from ._note import save_note # noqa +from ._preference_setting import register_preference_feedback_component # noqa __version__ = "0.13.0b1" From 098a607de492a552dadcae67ca80b38c912c3de9 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 19:11:03 +0900 Subject: [PATCH 27/38] fix by changed api, reloading --- optuna_dashboard/_app.py | 2 +- optuna_dashboard/_serializer.py | 4 - optuna_dashboard/ts/apiClient.ts | 19 ++- optuna_dashboard/ts/components/AppDrawer.tsx | 4 +- .../ts/components/PreferentialTrials.tsx | 154 ++++++++++++------ optuna_dashboard/ts/types/index.d.ts | 2 +- 6 files changed, 115 insertions(+), 70 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index abd717d0..5b1f7147 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -327,7 +327,7 @@ def create_app( return {"reason": "invalid request."} if component_type not in ["note", "artifact"]: response.status = 400 - return {"reason": "component_type must be either 'Note' or 'Artifact'."} + return {"reason": "component_type must be either 'note' or 'artifact'."} _register_preference_feedback_component( study_id=study_id, diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 8e3ffee2..5829f35d 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -165,10 +165,6 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets - if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: - serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] - if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: - serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] if serialized["is_preferential"]: serialized["feedback_component_type"] = system_attrs.get( _SYSTEM_ATTR_FEEDBACK_COMPONENT, {} diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 30eec71d..d2a786cf 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -83,6 +83,10 @@ const convertPreferenceHistory = ( } } +interface FeedbackComponentResponse { + type: string + artifact_key?: string +} interface StudyDetailResponse { name: string datetime_start: string @@ -101,8 +105,7 @@ interface StudyDetailResponse { preferences?: [number, number][] preference_history?: PreferenceHistoryResponce[] plotly_graph_objects: PlotlyGraphObject[] - feedback_component_type?: FeedbackComponentType - feedback_artifact_key?: string + feedback_component_type?: FeedbackComponentResponse skipped_trials?: number[] } @@ -139,10 +142,10 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, - feedback_component_type: res.data.feedback_component_type - ? (res.data.feedback_component_type as FeedbackComponentType) - : "Note", - feedback_artifact_key: res.data.feedback_artifact_key, + feedback_component_type: res.data.feedback_component_type?.type + ? (res.data.feedback_component_type.type as FeedbackComponentType) + : "note", + feedback_artifact_key: res.data.feedback_component_type?.artifact_key, preferences: res.data.preferences, preference_history: res.data.preference_history?.map( convertPreferenceHistory @@ -389,8 +392,8 @@ export const reportFeedbackComponentAPI = ( artifact_key?: string ): Promise => { return axiosInstance - .post(`/api/studies/${studyId}/component`, { - component_type: component_type, + .put(`/api/studies/${studyId}/preference_feedback_component_type`, { + type: component_type, artifact_key: artifact_key, }) .then(() => { diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 8740c68a..b9117707 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -287,7 +287,7 @@ export const AppDrawer: FC<{ - + - + diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 36cf9ab9..26ede44d 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -1,4 +1,4 @@ -import React, { FC, useEffect, useState } from "react" +import React, { FC, useEffect, useState, useMemo } from "react" import { Typography, Box, @@ -76,7 +76,7 @@ const SettingsPage: FC<{ const theme = useTheme() const actions = actionCreator() const [outputComponent, setOutputComponent] = useState( - studyDetail?.feedback_component_type ?? "Note" + studyDetail?.feedback_component_type ?? "note" ) const [outputArtifactKey, setOutputArtifactKey] = useState( studyDetail?.feedback_artifact_key ?? "" @@ -124,11 +124,11 @@ const SettingsPage: FC<{ setOutputComponent(e.target.value as FeedbackComponentType) }} > - Note - Artifact + Note + Artifact - {outputComponent === "Artifact" ? ( + {outputComponent === "artifact" ? ( { + if (componentId === undefined || componentId === "note") { + return trial.note.body !== "" + } + if (componentId === "artifact") { + const artifactId = trial?.user_attrs.find( + (a) => a.key === artifactKey + )?.value + const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) + return artifact !== undefined + } + return false +} + export const OutputContent: FC<{ trial: Trial artifact?: Artifact componentId?: FeedbackComponentType urlPath: string }> = ({ trial, artifact, componentId, urlPath }) => { - if ( - (componentId === undefined || componentId === "Note") && - trial.note.body !== "" - ) { + const note = useMemo(() => { return + }, [trial.note.body]) + if (componentId === undefined || componentId === "note") { + return note } - if (componentId === "Artifact" && artifact !== undefined) { + if (componentId === "artifact") { + if (artifact === undefined) { + return null + } return ( ) } - - return + return null } export const getArtifactUrlPath = ( @@ -222,7 +242,7 @@ const PreferentialTrial: FC<{ ? getArtifactUrlPath(studyDetail.id, trial?.trial_id, artifactId) : "" const is3dModel = - componentId === "Artifact" && + componentId === "artifact" && artifact !== undefined && isThreejsArtifact(artifact) @@ -242,6 +262,7 @@ const PreferentialTrial: FC<{ hideTrial() action.updatePreference(trial.study_id, candidates, trial.number) } + const isReady = isComparisonReady(trial, componentId, artifactKey) return ( Trial {trial.number} - {componentId === "Artifact" && artifact !== undefined ? ( + {componentId === "artifact" && artifact !== undefined ? ( - - - + {isReady ? ( + <> + + + + + ) : ( + + )} + + + ) } @@ -571,22 +550,51 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail={studyDetail} /> {detailTrial !== null && ( - { - setDetailTrial(null) - }} - > - - studyDetail.trials.find((t) => t.trial_id === trialId)?.state === - "Complete" ?? false - } - directions={[]} - objectiveNames={[]} - /> - + setDetailTrial(null)}> + + + + + + + studyDetail.trials.find((t) => t.trial_id === trialId) + ?.state === "Complete" ?? false + } + directions={[]} + objectiveNames={[]} + /> + + + )} {renderThreejsArtifactModal()} From 9a4f91ce07962cd7683c8867ef974f372bde7dfa Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 15 Sep 2023 19:30:58 +0900 Subject: [PATCH 36/38] split output component --- .../ts/components/PreferenceHistory.tsx | 5 ++-- .../ts/components/PreferentialGraph.tsx | 5 ++-- .../PreferentialOutputComponent.tsx | 26 ++++++++++++++++ .../ts/components/PreferentialTrials.tsx | 30 ++----------------- 4 files changed, 35 insertions(+), 31 deletions(-) create mode 100644 optuna_dashboard/ts/components/PreferentialOutputComponent.tsx diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 97a19dfe..e9c4948c 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -14,9 +14,10 @@ import Modal from "@mui/material/Modal" import { red } from "@mui/material/colors" import { TrialListDetail } from "./TrialList" -import { OutputContent, getArtifactUrlPath } from "./PreferentialTrials" +import { getArtifactUrlPath } from "./PreferentialTrials" import { formatDate } from "../dateUtil" import { useStudyDetailValue } from "../state" +import { PreferentialOutputComponent } from "./PreferentialOutputComponent" type TrialType = "worst" | "none" @@ -93,7 +94,7 @@ const CandidateTrial: FC<{ padding: theme.spacing(2), }} > - > = ({ data, isConnectable }) => { isConnectable={isConnectable} /> - = ({ trial, artifact, componentType, urlPath }) => { + const note = useMemo(() => { + return + }, [trial.note.body]) + if (componentType === undefined || componentType.output_type === "note") { + return note + } + if (componentType.output_type === "artifact") { + if (artifact === undefined) { + return null + } + return ( + + ) + } + return null +} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 120c317b..27f3a8d5 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -1,4 +1,4 @@ -import React, { FC, useEffect, useState, useMemo } from "react" +import React, { FC, useEffect, useState } from "react" import { Typography, Box, @@ -32,8 +32,7 @@ import { isThreejsArtifact, useThreejsArtifactModal, } from "./ThreejsArtifactViewer" -import { ArtifactCardMedia } from "./ArtifactCardMedia" -import { MarkdownRenderer } from "./Note" +import { PreferentialOutputComponent } from "./PreferentialOutputComponent" const SettingsPage: FC<{ studyDetail: StudyDetail @@ -169,29 +168,6 @@ const isComparisonReady = ( return false } -export const OutputContent: FC<{ - trial: Trial - artifact?: Artifact - componentType: FeedbackComponentType - urlPath: string -}> = ({ trial, artifact, componentType, urlPath }) => { - const note = useMemo(() => { - return - }, [trial.note.body]) - if (componentType === undefined || componentType.output_type === "note") { - return note - } - if (componentType.output_type === "artifact") { - if (artifact === undefined) { - return null - } - return ( - - ) - } - return null -} - export const getArtifactUrlPath = ( studyId: number, trialId: number, @@ -338,7 +314,7 @@ const PreferentialTrial: FC<{ > {isReady ? ( <> - Date: Tue, 19 Sep 2023 10:36:50 +0900 Subject: [PATCH 37/38] fix test --- python_tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 86f897c8..f9c56e4f 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -197,7 +197,7 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference_feedback_component", "PUT", - body=json.dumps({"type": "artifact", "artifact_key": "image"}), + body=json.dumps({"output_type": "artifact", "artifact_key": "image"}), content_type="application/json", ) self.assertEqual(status, 204) From e40f410264b35c5fc4f343309fdbbeabb53529d6 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 19 Sep 2023 13:38:42 +0900 Subject: [PATCH 38/38] fix by review --- optuna_dashboard/_serializer.py | 12 ++++++------ optuna_dashboard/ts/apiClient.ts | 6 ++---- optuna_dashboard/ts/components/AppDrawer.tsx | 4 ++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 016b7fb8..e1840df9 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -165,13 +165,13 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + serialized["feedback_component_type"] = system_attrs.get( + _SYSTEM_ATTR_FEEDBACK_COMPONENT, + { + "output_type": "note", + }, + ) if serialized["is_preferential"]: - serialized["feedback_component_type"] = system_attrs.get( - _SYSTEM_ATTR_FEEDBACK_COMPONENT, - { - "output_type": "note", - }, - ) serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) serialized["skipped_trials"] = skipped_trials diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index e552f773..60c461db 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -101,7 +101,7 @@ interface StudyDetailResponse { preferences?: [number, number][] preference_history?: PreferenceHistoryResponce[] plotly_graph_objects: PlotlyGraphObject[] - feedback_component_type?: FeedbackComponentType + feedback_component_type: FeedbackComponentType skipped_trials?: number[] } @@ -138,9 +138,7 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, - feedback_component_type: res.data.feedback_component_type ?? { - output_type: "note", - }, + feedback_component_type: res.data.feedback_component_type, preferences: res.data.preferences, preference_history: res.data.preference_history?.map( convertPreferenceHistory diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index b9117707..8740c68a 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -287,7 +287,7 @@ export const AppDrawer: FC<{ - + - +