diff --git a/docs/api.rst b/docs/api.rst index aadd2718..18f8bc72 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,6 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy + optuna_dashboard.register_preference_feedback_component Streamlit ----------------- diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 5bb3f301..ea2a8dbe 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -14,6 +14,7 @@ from ._form_widget import TextInputWidget # noqa from ._named_objectives import set_objective_names # noqa from ._note import get_note # noqa from ._note import save_note # noqa +from ._preference_setting import register_preference_feedback_component # noqa __version__ = "0.13.0b1" diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index e13334d3..812201ad 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,6 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preference_setting import _register_preference_feedback_component from ._preferential_history import NewHistory from ._preferential_history import PreferenceHistoryNotFound from ._preferential_history import remove_history @@ -313,6 +314,28 @@ def create_app( response.status = 204 return {} + @app.put("/api/studies//preference_feedback_component") + @json_api_view + def put_preference_feedback_component(study_id: int) -> dict[str, Any]: + try: + component_type = request.json.get("output_type", "") + artifact_key = request.json.get("artifact_key", None) + except ValueError: + response.status = 400 + return {"reason": "invalid request."} + if component_type not in ["note", "artifact"]: + response.status = 400 + return {"reason": "component_type must be either 'note' or 'artifact'."} + + _register_preference_feedback_component( + study_id=study_id, + storage=storage, + component_type=component_type, + artifact_key=artifact_key, + ) + response.status = 204 + return {} + @app.delete("/api/studies//preference/") @json_api_view def remove_preference(study_id: int, history_id: str) -> dict[str, Any]: diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py new file mode 100644 index 00000000..9f1e8dec --- /dev/null +++ b/optuna_dashboard/_preference_setting.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from typing import Any +from typing import TYPE_CHECKING + +from optuna.storages import BaseStorage + +from .preferential._study import PreferentialStudy + + +if TYPE_CHECKING: + from typing import Literal + + OUTPUT_COMPONENT_TYPE = Literal["note", "artifact"] + +_SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" + + +def _register_preference_feedback_component( + study_id: int, + storage: BaseStorage, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str | None = None, +) -> None: + value: dict[str, Any] = {"output_type": component_type} + if artifact_key is not None: + value["artifact_key"] = artifact_key + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_COMPONENT, + value=value, + ) + + +def register_preference_feedback_component( + study: PreferentialStudy, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str | None = None, +) -> None: + """Register a preference feedback component to the study. + + With this feature, you can change the component, displayed on the + human feedback pages. By default, the Markdown note (``component_type="note"``) + is displayed. If you specify ``component_type="artifact"``, the viewer for the + specified artifact file will be displayed. + Args: + study: + The study to register the preference feedback component. + component_type: + The component type, displayed on the human feedback pages + (default: ``"note"``). + user_attr_artifact_key: + This option is required when the ``component_type`` is ``"artifact"``. + The user attribute, which is specified this field, must contain the + ``artifact``id you want to display on the human feedback page. + """ + if component_type == "artifact": + assert ( + artifact_key is not None + ), "artifact_key must be specified when component_type is Artifact" + + _register_preference_feedback_component( + study_id=study._study._study_id, + storage=study._study._storage, + component_type=component_type, + artifact_key=artifact_key, + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 1f2b84bd..81544ea8 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,6 +15,7 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -164,6 +165,12 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + serialized["feedback_component_type"] = system_attrs.get( + _SYSTEM_ATTR_FEEDBACK_COMPONENT, + { + "output_type": "note", + }, + ) if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 8349b1a4..d023e741 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -1,8 +1,10 @@ from __future__ import annotations +import itertools import math from typing import Any from typing import Callable +from typing import cast import botorch.acquisition.analytic import botorch.models.model @@ -14,6 +16,7 @@ from gpytorch.likelihoods.gaussian_likelihood import Prior import numpy as np import optuna import optuna._transform +from optuna.distributions import CategoricalDistribution import torch from torch import Tensor @@ -310,7 +313,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): search_space: dict[str, optuna.distributions.BaseDistribution], ) -> dict[str, Any]: preferences = get_preferences(study.system_attrs) - if len(preferences) == 0: + if len(preferences) == 0 or len(search_space) == 0: return {} trials = study.get_trials(deepcopy=False) @@ -355,16 +358,33 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): best_f=torch.max(sampled_gp.posterior(params[:, None, :]).mean), ) - # TODO: Make it possible to apply it on categorical variables - candidates, _ = botorch.optim.optimize_acqf( - acq_function=acqf, - bounds=torch.from_numpy(trans.bounds.T), - q=1, - num_restarts=10, - raw_samples=512, - options={"batch_limit": 5, "maxiter": 200}, - sequential=True, - ) + # TODO: Make it possible to apply it on mixed search space + if all(isinstance(dist, CategoricalDistribution) for dist in search_space.values()): + all_param_combinations = itertools.product( + *[ + [(name, choice) for choice in cast(CategoricalDistribution, dist).choices] + for name, dist in search_space.items() + ] + ) + choices = torch.tensor( + np.array([trans.transform(dict(params)) for params in all_param_combinations]), + dtype=torch.float64, + ) + candidates, _ = botorch.optim.optimize_acqf_discrete( + acq_function=acqf, + choices=choices, + q=1, + ) + else: + candidates, _ = botorch.optim.optimize_acqf( + acq_function=acqf, + bounds=torch.from_numpy(trans.bounds.T), + q=1, + num_restarts=10, + raw_samples=512, + options={"batch_limit": 5, "maxiter": 200}, + sequential=True, + ) next_x = trans.untransform(candidates[0].detach().numpy()) return next_x diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index 1e75a3dc..a082d5fd 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -18,6 +18,7 @@ import { skipPreferentialTrialAPI, removePreferentialHistoryAPI, restorePreferentialHistoryAPI, + reportFeedbackComponentAPI, } from "./apiClient" import { graphVisibilityState, @@ -610,6 +611,27 @@ export const actionCreator = () => { console.log(err) }) } + const updateFeedbackComponent = ( + studyId: number, + compoennt_type: FeedbackComponentType + ) => { + reportFeedbackComponentAPI(studyId, compoennt_type) + .then(() => { + const newStudy = Object.assign({}, studyDetails[studyId]) + newStudy.feedback_component_type = compoennt_type + setStudyDetailState(studyId, newStudy) + }) + .catch((err) => { + const reason = err.response?.data.reason + enqueueSnackbar( + `Failed to report feedback component. Reason: ${reason}`, + { + variant: "error", + } + ) + console.log(err) + }) + } const removePreferentialHistory = (studyId: number, historyId: string) => { removePreferentialHistoryAPI(studyId, historyId) @@ -622,6 +644,7 @@ export const actionCreator = () => { }) .catch((err) => { const reason = err.response?.data.reason + enqueueSnackbar(`Failed to switch history. Reason: ${reason}`, { variant: "error", }) @@ -669,6 +692,7 @@ export const actionCreator = () => { skipPreferentialTrial, removePreferentialHistory, restorePreferentialHistory, + updateFeedbackComponent, } } diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 9f9cd8cb..d6b3c77a 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -99,6 +99,7 @@ interface StudyDetailResponse { preferences?: [number, number][] preference_history?: PreferenceHistoryResponce[] plotly_graph_objects: PlotlyGraphObject[] + feedback_component_type: FeedbackComponentType skipped_trial_numbers?: number[] } @@ -135,6 +136,7 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, + feedback_component_type: res.data.feedback_component_type, preferences: res.data.preferences, preference_history: res.data.preference_history?.map( convertPreferenceHistory @@ -395,3 +397,17 @@ export const restorePreferentialHistoryAPI = ( return }) } + +export const reportFeedbackComponentAPI = ( + studyId: number, + component_type: FeedbackComponentType +): Promise => { + return axiosInstance + .put( + `/api/studies/${studyId}/preference_feedback_component`, + component_type + ) + .then(() => { + return + }) +} diff --git a/optuna_dashboard/ts/components/BestTrialsCard.tsx b/optuna_dashboard/ts/components/BestTrialsCard.tsx index f228fe52..cb560921 100644 --- a/optuna_dashboard/ts/components/BestTrialsCard.tsx +++ b/optuna_dashboard/ts/components/BestTrialsCard.tsx @@ -32,22 +32,23 @@ export const BestTrialsCard: FC<{ header = `Best Trial (number=${bestTrial.number})` content = ( <> - {bestTrial.values === undefined || bestTrial.values.length === 1 ? ( - - {bestTrial.values} - - ) : ( - - Objective Values = [{bestTrial.values?.join(", ")}] - - )} + {!studyDetail?.is_preferential && + (bestTrial.values === undefined || bestTrial.values.length === 1 ? ( + + {bestTrial.values} + + ) : ( + + Objective Values = [{bestTrial.values?.join(", ")}] + + ))} Params = [ {bestTrial.params diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index db4df078..a29f84b8 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -16,9 +16,11 @@ import Modal from "@mui/material/Modal" import { red } from "@mui/material/colors" import { TrialListDetail } from "./TrialList" -import { MarkdownRenderer } from "./Note" +import { getArtifactUrlPath } from "./PreferentialTrials" import { formatDate } from "../dateUtil" import { actionCreator } from "../action" +import { useStudyDetailValue } from "../state" +import { PreferentialOutputComponent } from "./PreferentialOutputComponent" type TrialType = "worst" | "none" @@ -29,8 +31,24 @@ const CandidateTrial: FC<{ const theme = useTheme() const trialWidth = 300 const trialHeight = 300 + const studyDetail = useStudyDetailValue(trial.study_id) const [detailShown, setDetailShown] = useState(false) + if (studyDetail === null) { + return null + } + const componentType = studyDetail.feedback_component_type + const artifactId = + componentType.output_type === "artifact" + ? trial.user_attrs.find((a) => a.key === componentType.artifact_key) + ?.value + : undefined + const artifact = trial.artifacts.find((a) => a.artifact_id === artifactId) + const urlPath = + artifactId !== undefined + ? getArtifactUrlPath(trial.study_id, trial.trial_id, artifactId) + : "" + const cardComponentSx = { padding: 0, position: "relative", @@ -79,7 +97,12 @@ const CandidateTrial: FC<{ padding: theme.spacing(2), }} > - + {type === "worst" ? ( diff --git a/optuna_dashboard/ts/components/PreferentialGraph.tsx b/optuna_dashboard/ts/components/PreferentialGraph.tsx index f343cb06..e30974c1 100644 --- a/optuna_dashboard/ts/components/PreferentialGraph.tsx +++ b/optuna_dashboard/ts/components/PreferentialGraph.tsx @@ -1,4 +1,4 @@ -import React, { FC, useState, useCallback, useMemo, useEffect } from "react" +import React, { FC, useState, useCallback, useEffect } from "react" import { Card, CardContent, @@ -7,7 +7,6 @@ import { Box, Chip, } from "@mui/material" -import { MarkdownRenderer } from "./Note" import ReactFlow, { Node, NodeProps, @@ -24,6 +23,10 @@ import "reactflow/dist/style.css" import ELK from "elkjs/lib/elk.bundled.js" import { ElkNode } from "elkjs/lib/elk-api.js" +import { useStudyDetailValue } from "../state" +import { getArtifactUrlPath } from "./PreferentialTrials" +import { PreferentialOutputComponent } from "./PreferentialOutputComponent" + const elk = new ELK() const nodeWidth = 400 const nodeHeight = 300 @@ -39,10 +42,22 @@ const GraphNode: FC> = ({ data, isConnectable }) => { if (trial === undefined) { return null } - const noteBody = trial.note.body - const noteFC = useMemo(() => { - return - }, [noteBody]) + const studyDetail = useStudyDetailValue(trial.study_id) + const componentType = studyDetail?.feedback_component_type + if (componentType === undefined) { + return null + } + const artifactId = + componentType.output_type === "artifact" + ? trial.user_attrs.find((a) => a.key === componentType.artifact_key) + ?.value + : undefined + const artifact = trial.artifacts.find((a) => a.artifact_id === artifactId) + const urlPath = + artifactId !== undefined + ? getArtifactUrlPath(trial.study_id, trial.trial_id, artifactId) + : "" + return ( > = ({ data, isConnectable }) => { style={{ background: "#555" }} isConnectable={isConnectable} /> - {noteFC} + + + = ({ trial, artifact, componentType, urlPath }) => { + const note = useMemo(() => { + return + }, [trial.note.body]) + if (componentType === undefined || componentType.output_type === "note") { + return note + } + if (componentType.output_type === "artifact") { + if (artifact === undefined) { + return null + } + return ( + + ) + } + return null +} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index feb845fc..64429451 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -1,4 +1,4 @@ -import React, { FC, useState } from "react" +import React, { FC, useEffect, useState } from "react" import { Typography, Box, @@ -6,33 +6,214 @@ import { Card, CardContent, CardActions, - CardActionArea, + Button, + MenuItem, + Select, + FormControl, + FormLabel, + Modal, CircularProgress, + Dialog, + DialogTitle, + DialogContent, + DialogActions, } from "@mui/material" -import ClearIcon from "@mui/icons-material/Clear" import IconButton from "@mui/material/IconButton" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import ReplayIcon from "@mui/icons-material/Replay" -import Modal from "@mui/material/Modal" import { red } from "@mui/material/colors" import UndoIcon from "@mui/icons-material/Undo" +import ClearIcon from "@mui/icons-material/Clear" +import SettingsIcon from "@mui/icons-material/Settings" +import FullscreenIcon from "@mui/icons-material/Fullscreen" import { actionCreator } from "../action" import { TrialListDetail } from "./TrialList" -import { MarkdownRenderer } from "./Note" +import { + isThreejsArtifact, + useThreejsArtifactModal, +} from "./ThreejsArtifactViewer" +import { PreferentialOutputComponent } from "./PreferentialOutputComponent" + +const SettingsPage: FC<{ + studyDetail: StudyDetail + settingShown: boolean + setSettingShown: (flag: boolean) => void +}> = ({ studyDetail, settingShown, setSettingShown }) => { + const actions = actionCreator() + const [outputComponentType, setOutputComponentType] = useState( + studyDetail.feedback_component_type.output_type + ) + const [artifactKey, setArtifactKey] = useState( + studyDetail.feedback_component_type.output_type === "artifact" + ? studyDetail.feedback_component_type.artifact_key + : undefined + ) + useEffect(() => { + setOutputComponentType(studyDetail.feedback_component_type.output_type) + }, [studyDetail.feedback_component_type.output_type]) + useEffect(() => { + if (studyDetail.feedback_component_type.output_type === "artifact") { + setArtifactKey(studyDetail.feedback_component_type.artifact_key) + } + }, [ + studyDetail.feedback_component_type.output_type === "artifact" + ? studyDetail.feedback_component_type.artifact_key + : undefined, + ]) + const onClose = () => { + setSettingShown(false) + } + const onApply = () => { + setSettingShown(false) + const outputComponent: FeedbackComponentType = + outputComponentType === "note" + ? ({ output_type: "note" } as FeedbackComponentNote) + : ({ + output_type: "artifact", + artifact_key: artifactKey, + } as FeedbackComponentArtifact) + actions.updateFeedbackComponent(studyDetail.id, outputComponent) + } + + return ( + + Settings + + + Output Component: + + + {outputComponentType === "artifact" ? ( + + + User Attribute Key Corresponding to Output Artifact Id: + + + + ) : null} + + + + + + + ) +} + +const isComparisonReady = ( + trial: Trial, + componentType: FeedbackComponentType +): boolean => { + if (componentType === undefined || componentType.output_type === "note") { + return trial.note.body !== "" + } + if (componentType.output_type === "artifact") { + const artifactId = trial?.user_attrs.find( + (a) => a.key === componentType.artifact_key + )?.value + const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) + return artifact !== undefined + } + return false +} + +export const getArtifactUrlPath = ( + studyId: number, + trialId: number, + artifactId: string +): string => { + return `/artifacts/${studyId}/${trialId}/${artifactId}` +} const PreferentialTrial: FC<{ trial?: Trial + studyDetail: StudyDetail candidates: number[] hideTrial: () => void -}> = ({ trial, candidates, hideTrial }) => { + openDetailTrial: () => void + openThreejsArtifactModal: (urlPath: string, artifact: Artifact) => void +}> = ({ + trial, + studyDetail, + candidates, + hideTrial, + openDetailTrial, + openThreejsArtifactModal, +}) => { const theme = useTheme() const action = actionCreator() - const trialWidth = 500 + const [buttonHover, setButtonHover] = useState(false) + const trialWidth = 400 const trialHeight = 300 - const [detailShown, setDetailShown] = useState(false) + const componentType = studyDetail.feedback_component_type + const artifactId = + componentType.output_type === "artifact" + ? trial?.user_attrs.find((a) => a.key === componentType.artifact_key) + ?.value + : undefined + const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) + const urlPath = + trial !== undefined && artifactId !== undefined + ? getArtifactUrlPath(studyDetail.id, trial?.trial_id, artifactId) + : "" + const is3dModel = + componentType.output_type === "artifact" && + artifact !== undefined && + isThreejsArtifact(artifact) - if (trial == undefined) { + if (trial === undefined) { return ( { + hideTrial() + action.updatePreference(trial.study_id, candidates, trial.number) + } + const isReady = isComparisonReady(trial, componentType) return ( - Trial {trial.number} + + Trial {trial.number} + {componentType.output_type === "artifact" && + artifact !== undefined ? ( + + {`(${artifact.filename})`} + + ) : null} + + {is3dModel ? ( + { + openThreejsArtifactModal(urlPath, artifact) + }} + > + + + ) : null} setDetailShown(true)} + onClick={openDetailTrial} aria-label="show detail" > - - { - hideTrial() - action.updatePreference(trial.study_id, candidates, trial.number) - }} - sx={{ - padding: 0, - position: "relative", - overflow: "hidden", - "::before": { - content: '""', - position: "absolute", - top: 0, - left: 0, - width: "100%", - height: "100%", - backgroundColor: - theme.palette.mode === "dark" ? "white" : "black", - opacity: 0, - zIndex: 1, - transition: "opacity 0.3s ease-out", - }, - ":hover::before": { - opacity: 0.2, - }, - }} - > - - {trial.note.body !== "" ? ( - - ) : ( - - )} - - - { + if (e.shiftKey) onFeedback() + }} + sx={{ + position: "relative", + padding: theme.spacing(2), + overflow: "hidden", + minHeight: theme.spacing(20), + }} + > + {isReady ? ( + <> + + + + + ) : ( + - - - setDetailShown(false)}> - - - isBestTrial} - directions={[]} - objectiveNames={[]} - /> - - - + )} + + ) } @@ -186,13 +399,17 @@ type DisplayTrials = { export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail, }) => { + const theme = useTheme() + const action = actionCreator() const [undoHistoryId, setUndoHistoryId] = useState(null) + const [openThreejsArtifactModal, renderThreejsArtifactModal] = + useThreejsArtifactModal() const [displayTrials, setDisplayTrials] = useState({ display: [], clicked: [], }) - const theme = useTheme() - const action = actionCreator() + const [settingShown, setSettingShown] = useState(false) + const [detailTrial, setDetailTrial] = useState(null) if (studyDetail === null || !studyDetail.is_preferential) { return null @@ -274,34 +491,124 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ > Which trial is the worst? - { - if (latestHistoryId === null) { - return - } - setUndoHistoryId(latestHistoryId) - action.removePreferentialHistory(studyDetail.id, latestHistoryId) - }} + - - + + + - {displayTrials.display.map((t, index) => ( - trial.number === t)} - candidates={displayTrials.display.filter((n) => n !== -1)} - hideTrial={() => { - hideTrial(t) - }} - /> - ))} + {displayTrials.display.map((t, index) => { + const trial = activeTrials.find((trial) => trial.number === t) + const candidates = displayTrials.display.filter( + (n) => + n !== -1 && + isComparisonReady( + studyDetail.trials[n], + studyDetail.feedback_component_type + ) + ) + return ( + hideTrial(t)} + openDetailTrial={() => setDetailTrial(t)} + openThreejsArtifactModal={openThreejsArtifactModal} + /> + ) + })} + + {detailTrial !== null && ( + setDetailTrial(null)}> + + + setDetailTrial(null)} + > + + + + studyDetail.trials.find((t) => t.trial_id === trialId) + ?.state === "Complete" ?? false + } + directions={[]} + objectiveNames={[]} + /> + + + + )} + {renderThreejsArtifactModal()} ) } diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index a51a88d9..3e883e14 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -187,6 +187,17 @@ type PlotlyGraphObject = { graph_object: string } +type FeedbackComponentNote = { + output_type: "note" +} + +type FeedbackComponentArtifact = { + output_type: "artifact" + artifact_key: string +} + +type FeedbackComponentType = FeedbackComponentArtifact | FeedbackComponentNote + type StudyDetail = { id: number name: string @@ -203,6 +214,7 @@ type StudyDetail = { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + feedback_component_type: FeedbackComponentType preferences?: [number, number][] preference_history?: PreferenceHistory[] plotly_graph_objects: PlotlyGraphObject[] diff --git a/python_tests/test_api.py b/python_tests/test_api.py index afe5a31c..f9c56e4f 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,6 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study +from optuna_dashboard._preference_setting import register_preference_feedback_component from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history @@ -183,6 +184,36 @@ class APITestCase(TestCase): ) self.assertEqual(status, 400) + def test_change_component(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + register_preference_feedback_component(study, "note") + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference_feedback_component", + "PUT", + body=json.dumps({"output_type": "artifact", "artifact_key": "image"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"]["output_type"] == "artifact" + assert study_detail["feedback_component_type"]["artifact_key"] == "image" + def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py new file mode 100644 index 00000000..5499e134 --- /dev/null +++ b/python_tests/test_preference_setting.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from unittest import TestCase + +import optuna +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT +from optuna_dashboard._preference_setting import register_preference_feedback_component +from optuna_dashboard.preferential._study import PreferentialStudy + + +class FeedbackSettingTestCase(TestCase): + def test_widget_to_dict_from_dict(self) -> None: + study = PreferentialStudy(optuna.create_study()) + register_preference_feedback_component(study, "artifact", "image_key") + system_attrs = study._study.system_attrs + feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) + assert "output_type" in feedback_type + assert feedback_type["output_type"] == "artifact" + assert "artifact_key" in feedback_type + assert feedback_type["artifact_key"] == "image_key"