diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index f582d5e5..0b69dfc7 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,6 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials from ._preferential_history import report_history +from ._preferential_history import switching_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -296,6 +297,21 @@ def create_app( response.status = 204 return {} + @app.put("/api/studies//preference/") + @json_api_view + def switch_preference(study_id: int, history_uuid: str) -> dict[str, Any]: + try: + enable = request.json.get("enable", None) + if enable is None or not isinstance(enable, bool): + raise ValueError + except ValueError: + response.status = 400 + return {"reason": "Invalid request."} + switching_history(study_id, storage, history_uuid, enable) + + response.status = 204 + return {} + @app.post("/api/trials//tell") @json_api_view def tell_trial(trial_id: int) -> dict[str, Any]: diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 5c930932..dff10885 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -1,8 +1,8 @@ from __future__ import annotations from dataclasses import dataclass +from dataclasses import field from datetime import datetime -import json from typing import Any from typing import Literal from typing import TYPE_CHECKING @@ -10,6 +10,8 @@ import uuid from optuna.storages import BaseStorage +from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE +from .preferential._system_attrs import get_preference from .preferential._system_attrs import report_preferences @@ -28,7 +30,7 @@ if TYPE_CHECKING: ) -@dataclass(frozen=True) +@dataclass class ChooseWorstHistory: mode: Literal["ChooseWorst"] uuid: str @@ -36,6 +38,8 @@ class ChooseWorstHistory: timestamp: datetime candidates: list[int] # a list of trial number clicked: int # The worst trial number in the candidates. + evacuated_preference: list[tuple[int, int]] = field(default_factory=list) + # When undo the preference, this is used. Otherwise, this must be empty. def to_dict(self) -> dict[str, Any]: return { @@ -45,6 +49,8 @@ class ChooseWorstHistory: "timestamp": self.timestamp.isoformat(), "candidates": self.candidates, "clicked": self.clicked, + "enabled": len(self.evacuated_preference) == 0, + "evacuated_preference": self.evacuated_preference, } @@ -87,31 +93,65 @@ def report_history( storage.set_study_system_attr( study_id=study_id, key=key, - value=json.dumps(history.to_dict()), + value=history.to_dict(), ) -def serialize_preference_history( +def _load_preference_history(value: Any) -> History: + choice: dict[str, Any] = value + if choice["mode"] == "ChooseWorst": + return ChooseWorstHistory( + mode="ChooseWorst", + uuid=choice["uuid"], + preference_uuid=choice["preference_uuid"], + timestamp=datetime.fromisoformat(choice["timestamp"]), + candidates=choice["candidates"], + clicked=choice["clicked"], + evacuated_preference=choice["evacuated_preference"], + ) + else: + assert False, f"Unknown mode: {choice['mode']}" + + +def load_preference_history( + uuid: str, + system_attrs: dict[str, Any], +) -> History: + value = system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, []) + return _load_preference_history(value) + + +def serialize_preference_histories( system_attrs: dict[str, Any], ) -> list[dict[str, Any]]: histories: list[History] = [] for k, v in system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): continue - choice: dict[str, Any] = json.loads(v) - if choice["mode"] == "ChooseWorst": - histories.append( - ChooseWorstHistory( - mode="ChooseWorst", - uuid=choice["uuid"], - preference_uuid=choice["preference_uuid"], - timestamp=datetime.fromisoformat(choice["timestamp"]), - candidates=choice["candidates"], - clicked=choice["clicked"], - ) - ) - else: - assert False, f"Unknown mode: {choice['mode']}" + histories.append(_load_preference_history(v)) histories.sort(key=lambda c: c.timestamp) return [history.to_dict() for history in histories] + + +def switching_history(study_id: int, storage: BaseStorage, uuid: str, enable: bool) -> None: + system_attrs = storage.get_study_system_attrs(study_id) + history = load_preference_history(uuid, system_attrs) + preference = get_preference(study_id, storage, history.preference_uuid) + print(history, preference, enable) + if enable and (len(preference) > 0 or len(history.evacuated_preference) == 0): + return + if (not enable) and (len(preference) == 0 or len(history.evacuated_preference) > 0): + return + history.evacuated_preference, preference = preference, history.evacuated_preference + print(history.to_dict(), preference) + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_PREFIX_HISTORY + history.uuid, + value=history.to_dict(), + ) + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_PREFIX_PREFERENCE + history.preference_uuid, + value=preference, + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 19ceb19b..fc040424 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -14,7 +14,7 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names -from ._preferential_history import serialize_preference_history +from ._preferential_history import serialize_preference_histories from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -157,7 +157,7 @@ def serialize_study_detail( if form_widgets: serialized["form_widgets"] = form_widgets if serialized["is_preferential"]: - serialized["preference_history"] = serialize_preference_history(system_attrs) + serialized["preference_history"] = serialize_preference_histories(system_attrs) return serialized diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 00347d7e..4e3f55f5 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -34,6 +34,11 @@ def report_preferences( return preference_uuid +def get_preference(study_id: int, storage: BaseStorage, uuid: str) -> list[tuple[int, int]]: + system_attrs = storage.get_study_system_attrs(study_id) + return system_attrs.get(_SYSTEM_ATTR_PREFIX_PREFERENCE + uuid, []) # type: ignore + + def get_preferences( study_id: int, storage: BaseStorage, diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index e1896178..f42a17bb 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -16,6 +16,7 @@ import { deleteArtifactAPI, reportPreferenceAPI, skipPreferentialTrialAPI, + switchPreferentialHistoryAPI, } from "./apiClient" import { graphVisibilityState, @@ -609,6 +610,20 @@ export const actionCreator = () => { }) } + const switchPreferentialHistory = ( + studyId: number, + historyUuid: string, + enable: boolean + ) => { + switchPreferentialHistoryAPI(studyId, historyUuid, enable).catch((err) => { + const reason = err.response?.data.reason + enqueueSnackbar(`Failed to switch history. Reason: ${reason}`, { + variant: "error", + }) + console.log(err) + }) + } + return { updateAPIMeta, updateStudyDetail, @@ -630,6 +645,7 @@ export const actionCreator = () => { saveTrialUserAttrs, updatePreference, skipPreferentialTrial, + switchPreferentialHistory, } } diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 9c72f739..6886f8b4 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -61,6 +61,7 @@ interface PreferenceChoiceResponce { clicked: number mode: PreferenceFeedbackMode timestamp: string + enabled: boolean } const convertPreferenceChoice = ( @@ -72,6 +73,7 @@ const convertPreferenceChoice = ( clicked: res.clicked, feedback_mode: res.mode, timestamp: new Date(res.timestamp), + enabled: res.enabled, } } @@ -362,3 +364,17 @@ export const skipPreferentialTrialAPI = ( return }) } + +export const switchPreferentialHistoryAPI = ( + studyId: number, + historyUuid: string, + enable: boolean +): Promise => { + return axiosInstance + .put(`/api/studies/${studyId}/preference/${historyUuid}`, { + enable: enable, + }) + .then(() => { + return + }) +} diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index e95a4cb1..76725faa 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -9,12 +9,15 @@ import { } from "@mui/material" import ClearIcon from "@mui/icons-material/Clear" import IconButton from "@mui/material/IconButton" +import UndoIcon from "@mui/icons-material/Undo" +import RedoIcon from "@mui/icons-material/Redo" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import Modal from "@mui/material/Modal" import { TrialListDetail } from "./TrialList" import { MarkdownRenderer } from "./Note" import { red } from "@mui/material/colors" +import { actionCreator } from "../action" type TrialType = "worst" | "none" @@ -133,24 +136,52 @@ const CandidateTrial: FC<{ ) } -const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ - choice, - trials, -}) => { +const ChoiceTrials: FC<{ + choice: PreferenceChoice + trials: Trial[] + study_id: number +}> = ({ choice, trials, study_id }) => { const theme = useTheme() const worst_trials = new Set([choice.clicked]) + const actions = actionCreator() + const handleUndo = () => { + actions.switchPreferentialHistory(study_id, choice.uuid, false) + } + const handleRedo = () => { + actions.switchPreferentialHistory(study_id, choice.uuid, true) + } return ( - - {choice.timestamp.toISOString()} - + + {choice.timestamp.toLocaleString()} + + + + + + + + = ({ key={choice.uuid} choice={choice} trials={studyDetail.trials} + study_id={studyDetail.id} /> ))} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index da90ad1d..26c08d38 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -215,4 +215,5 @@ type PreferenceChoice = { clicked: number feedback_mode: PreferenceFeedbackMode timestamp: Date + enabled: boolean } diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 64591783..85f25374 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import Callable from optuna_dashboard._preferential_history import report_history -from optuna_dashboard._serializer import serialize_preference_history +from optuna_dashboard._serializer import serialize_preference_histories from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE @@ -40,7 +40,7 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) "clicked": 0, }, ) - history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + history = serialize_preference_histories(storage.get_study_system_attrs(study_id)) sys_attrs = storage.get_study_system_attrs(study_id) assert len(history) == 2 assert history[0]["candidates"] == [0, 1, 2]