From aed5bf2929c9fa6bda5714bc04ee59e4b83b929e Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 29 Aug 2023 15:41:44 +0900 Subject: [PATCH 01/30] add preference history --- optuna_dashboard/_app.py | 16 +- optuna_dashboard/_serializer.py | 20 +- optuna_dashboard/preferential/_history.py | 63 +++++ .../preferential/_system_attrs.py | 6 +- optuna_dashboard/ts/apiClient.ts | 38 ++- optuna_dashboard/ts/components/App.tsx | 9 + optuna_dashboard/ts/components/AppDrawer.tsx | 31 ++- .../ts/components/PreferenceHistory.tsx | 218 ++++++++++++++++++ .../ts/components/StudyDetail.tsx | 3 + optuna_dashboard/ts/types/index.d.ts | 10 + 10 files changed, 401 insertions(+), 13 deletions(-) create mode 100644 optuna_dashboard/preferential/_history.py create mode 100644 optuna_dashboard/ts/components/PreferenceHistory.tsx diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 2401f3f0..e18f37fc 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -8,6 +8,7 @@ from typing import Any from typing import Optional from typing import Union import warnings +from datetime import datetime from bottle import Bottle from bottle import redirect @@ -40,7 +41,8 @@ from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._study import get_best_trials as get_best_preferential_trials -from .preferential._system_attrs import report_preferences +from .preferential._history import FeedbackMode +from .preferential._history import report_choice if typing.TYPE_CHECKING: @@ -268,17 +270,17 @@ def create_app( @json_api_view def post_preference(study_id: int) -> dict[str, Any]: try: - best_trials = [int(d) for d in request.json.get("best_trials", [])] - worst_trials = [int(d) for d in request.json.get("worst_trials", [])] + candidate_trials = [int(d) for d in request.json.get("candidate_trials", [])] + preferences = [(int(d[0]), int(d[1])) for d in request.json.get("preferentials", [])] + mode = FeedbackMode[request.json.get("mode", "auto").upper()] except ValueError: response.status = 400 - return {"reason": "best_trials and worst_trials must be an array of integers."} - if len(best_trials) == 0 or len(worst_trials) == 0: + return {"reason": "Invalid request."} + if len(preferences) == 0: response.status = 400 # Bad request return {"reason": "You need to set best_trials and worst_trials"} - preferences = [(best, worst) for best in best_trials for worst in worst_trials] - report_preferences(study_id, storage, preferences) + report_choice(study_id, storage, candidate_trials, preferences, mode, datetime.now()) response.status = 204 return {} diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 0fa04124..372cf458 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -16,7 +16,8 @@ from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY - +from .preferential._history import Choice, _SYSTEM_ATTR_PREFIX_HISTORY +from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE if TYPE_CHECKING: from typing import Literal @@ -155,6 +156,8 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + if serialized["is_preferential"]: + serialized["preference_history"] = serialize_preference_history(system_attrs) return serialized @@ -326,3 +329,18 @@ def serialize_search_space( } ) return serialized + + +def serialize_preference_history( + system_attrs: dict[str, Any], +) -> list[dict[str, Any]]: + history: list[Choice] = [] + for k, v in system_attrs.items(): + if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): + continue + choice: dict[str, Any] = json.loads(v) + choice["preferences"] = system_attrs.get( + _SYSTEM_ATTR_PREFIX_PREFERENCE + choice["preference_uuid"], [] + ) + history.append(choice) + return history diff --git a/optuna_dashboard/preferential/_history.py b/optuna_dashboard/preferential/_history.py new file mode 100644 index 00000000..15a3ceb2 --- /dev/null +++ b/optuna_dashboard/preferential/_history.py @@ -0,0 +1,63 @@ +from enum import Enum +from datetime import datetime +import uuid +import json +from dataclasses import dataclass, asdict +from typing import Any +from json import JSONEncoder + +from optuna.storages import BaseStorage + +from ._system_attrs import report_preferences +from .._storage import get_study_summary + + +_SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" + + +class FeedbackMode(Enum): + CHOOSE_WORST = 0 + AUTO = 10 + + +@dataclass +class Choice: + uuid: str + candidate_trials: list[int] + preference_uuid: str + feedback_mode: FeedbackMode + timestamp: datetime + + +class Encoder(JSONEncoder): + def default(self, o): + if isinstance(o, FeedbackMode): + return o.name + if isinstance(o, Choice): + return asdict(o) + if isinstance(o, datetime): + return o.isoformat() + return super().default(o) + + +def report_choice( + study_id: int, + storage: BaseStorage, + candidate_trials: list[int], + preferences: list[tuple[int, int]], + feedback_mode: FeedbackMode, + timestamp: datetime, +): + choice = Choice( + uuid=str(uuid.uuid4()), + candidate_trials=candidate_trials, + preference_uuid=report_preferences(study_id, storage, preferences), + feedback_mode=feedback_mode, + timestamp=timestamp, + ) + key = _SYSTEM_ATTR_PREFIX_HISTORY + choice.uuid + storage.set_study_system_attr( + study_id=study_id, + key=key, + value=json.dumps(choice, cls=Encoder), + ) diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index fdd9db35..c9dc72c7 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -15,8 +15,9 @@ def report_preferences( study_id: int, storage: BaseStorage, preferences: list[tuple[int, int]], -) -> None: - key = _SYSTEM_ATTR_PREFIX_PREFERENCE + str(uuid.uuid4()) +) -> str: + preference_uuid = str(uuid.uuid4()) + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_uuid storage.set_study_system_attr( study_id=study_id, key=key, @@ -30,6 +31,7 @@ def report_preferences( trial_id = trials[number]._trial_id if trials[number].state != TrialState.COMPLETE: storage.set_trial_state_values(trial_id, TrialState.COMPLETE, values) + return preference_uuid def get_preferences( diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 5068dc3e..70bf2ef5 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -55,6 +55,26 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } +interface PreferenceChoiceResponce { + uuid: string + candidate_trials: number[] + preferences: number[][] + feedback_mode: PreferenceFeedbackMode + timestamp: string +} + +const convertPreferenceChoice = ( + res: PreferenceChoiceResponce +): PreferenceChoice => { + return { + uuid: res.uuid, + candidate_trials: res.candidate_trials, + preferences: res.preferences, + feedback_mode: res.feedback_mode, + timestamp: new Date(res.timestamp), + } +} + interface StudyDetailResponse { name: string datetime_start: string @@ -70,6 +90,7 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + preference_history?: PreferenceChoiceResponce[] } export const getStudyDetailAPI = ( @@ -105,6 +126,9 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, + preference_history: res.data.preference_history?.map( + convertPreferenceChoice + ), } }) } @@ -319,8 +343,18 @@ export const reportPreferenceAPI = ( ): Promise => { return axiosInstance .post(`/api/studies/${studyId}/preference`, { - best_trials: best_trials, - worst_trials: worst_trials, + candidate_trials: best_trials.concat(worst_trials), + preferentials: best_trials.reduce( + (prev, best_trial_id) => + prev.concat( + worst_trials.map((worst_trial_id) => [ + best_trial_id, + worst_trial_id, + ]) + ), + [] + ), + mode: "choose_worst", }) .then(() => { return diff --git a/optuna_dashboard/ts/components/App.tsx b/optuna_dashboard/ts/components/App.tsx index 78015049..e904ae30 100644 --- a/optuna_dashboard/ts/components/App.tsx +++ b/optuna_dashboard/ts/components/App.tsx @@ -96,6 +96,15 @@ export const App: FC = () => { /> } /> + + } + /> } diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 79f7d615..9dd4bd36 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -34,12 +34,19 @@ import GitHubIcon from "@mui/icons-material/GitHub" import OpenInNewIcon from "@mui/icons-material/OpenInNew" import QueryStatsIcon from "@mui/icons-material/QueryStats" import ThumbUpAltIcon from "@mui/icons-material/ThumbUpAlt" +import HistoryIcon from "@mui/icons-material/History" import { Switch } from "@mui/material" import { actionCreator } from "../action" const drawerWidth = 240 -export type PageId = "top" | "analytics" | "trialTable" | "trialList" | "note" +export type PageId = + | "top" + | "analytics" + | "trialTable" + | "trialList" + | "note" + | "preferenceHistory" const openedMixin = (theme: Theme): CSSObject => ({ width: drawerWidth, @@ -204,6 +211,28 @@ export const AppDrawer: FC<{ /> + {isPreferential && ( + + + + + + + + + )} {!isPreferential && ( = ({ trial, type }) => { + const theme = useTheme() + const trialWidth = 500 + const trialHeight = 300 + const [detailShown, setDetailShown] = useState(false) + + let cardComponentSx = { + padding: 0, + position: "relative", + overflow: "hidden", + "::before": {}, + } + if (type !== "none") { + cardComponentSx["::before"] = { + content: '""', + position: "absolute", + top: 0, + left: 0, + width: "100%", + height: "100%", + backgroundColor: theme.palette.mode === "dark" ? "white" : "black", + opacity: 0.2, + zIndex: 1, + transition: "opacity 0.3s ease-out", + } + } + + return ( + + + Trial {trial.number} + setDetailShown(true)} + aria-label="show detail" + > + + + + + + + + + {type === "worst" ? ( + + ) : null} + + setDetailShown(false)}> + + + false} + directions={[]} + objectiveNames={[]} + /> + + + + + ) +} + +const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ + choice, + trials, +}) => { + const theme = useTheme() + const worst_trials = new Set(choice.preferences.map((pair) => pair[1])) + console.log( + worst_trials, + choice.candidate_trials, + choice.preferences, + choice.preferences.map((pair) => pair[1]) + ) + + return ( + + + {choice.timestamp.toISOString()} + + + {choice.candidate_trials.map((trial_num, index) => ( + + ))} + + + ) +} + +export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({ + studyDetail, +}) => { + if ( + studyDetail === null || + !studyDetail.is_preferential || + studyDetail.preference_history === undefined + ) { + return null + } + const theme = useTheme() + + if (studyDetail.preference_history.length === 0) { + return ( + + No feedback history + + ) + } + + return ( + + {studyDetail.preference_history.map((choice) => ( + + ))} + + ) +} diff --git a/optuna_dashboard/ts/components/StudyDetail.tsx b/optuna_dashboard/ts/components/StudyDetail.tsx index 738166b5..c3f9da0a 100644 --- a/optuna_dashboard/ts/components/StudyDetail.tsx +++ b/optuna_dashboard/ts/components/StudyDetail.tsx @@ -30,6 +30,7 @@ import { GraphEdf } from "./GraphEdf" import { TrialList } from "./TrialList" import { StudyHistory } from "./StudyHistory" import { PreferentialTrials } from "./PreferentialTrials" +import { PreferenceHistory } from "./PreferenceHistory" interface ParamTypes { studyId: string @@ -172,6 +173,8 @@ export const StudyDetail: FC<{ /> ) + } else if (page == "preferenceHistory") { + content = } const toolbar = ( diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 1720cc6b..9e8e408f 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -12,6 +12,7 @@ type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type TrialStateFinished = "Complete" | "Fail" | "Pruned" type StudyDirection = "maximize" | "minimize" | "not_set" +type PreferenceFeedbackMode = "choose_worst" | "auto" type FloatDistribution = { type: "FloatDistribution" @@ -197,6 +198,7 @@ type StudyDetail = { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + preference_history?: PreferenceChoice[] } type StudyDetails = { @@ -206,3 +208,11 @@ type StudyDetails = { type StudyParamImportance = { [study_id: string]: ParamImportance[][] } + +type PreferenceChoice = { + uuid: string + candidate_trials: number[] + preferences: number[][] + feedback_mode: PreferenceFeedbackMode + timestamp: Date +} From 9564731ce97a5ee2a94ca9b18ecb83cf06ce0d9c Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 29 Aug 2023 16:58:51 +0900 Subject: [PATCH 02/30] add test and fix by lint --- optuna_dashboard/_app.py | 6 +-- optuna_dashboard/_serializer.py | 6 ++- optuna_dashboard/preferential/_history.py | 28 +++++----- .../ts/components/PreferenceHistory.tsx | 2 +- python_tests/preferential/test_history.py | 51 +++++++++++++++++++ python_tests/test_api.py | 8 ++- 6 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 python_tests/preferential/test_history.py diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index e18f37fc..bf32cb23 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import functools import logging import os @@ -8,7 +9,6 @@ from typing import Any from typing import Optional from typing import Union import warnings -from datetime import datetime from bottle import Bottle from bottle import redirect @@ -39,10 +39,10 @@ from ._storage_url import get_storage from .artifact._backend import delete_all_artifacts from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store -from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY -from .preferential._study import get_best_trials as get_best_preferential_trials from .preferential._history import FeedbackMode from .preferential._history import report_choice +from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY +from .preferential._study import get_best_trials as get_best_preferential_trials if typing.TYPE_CHECKING: diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 372cf458..741385ca 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,10 +15,11 @@ from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names from .artifact._backend import list_trial_artifacts +from .preferential._history import _SYSTEM_ATTR_PREFIX_HISTORY from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY -from .preferential._history import Choice, _SYSTEM_ATTR_PREFIX_HISTORY from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE + if TYPE_CHECKING: from typing import Literal from typing import TypedDict @@ -334,7 +335,7 @@ def serialize_search_space( def serialize_preference_history( system_attrs: dict[str, Any], ) -> list[dict[str, Any]]: - history: list[Choice] = [] + history: list[dict[str, Any]] = [] for k, v in system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): continue @@ -343,4 +344,5 @@ def serialize_preference_history( _SYSTEM_ATTR_PREFIX_PREFERENCE + choice["preference_uuid"], [] ) history.append(choice) + history.sort(key=lambda c: c["timestamp"]) return history diff --git a/optuna_dashboard/preferential/_history.py b/optuna_dashboard/preferential/_history.py index 15a3ceb2..4170ee21 100644 --- a/optuna_dashboard/preferential/_history.py +++ b/optuna_dashboard/preferential/_history.py @@ -1,15 +1,15 @@ -from enum import Enum +from dataclasses import asdict +from dataclasses import dataclass from datetime import datetime -import uuid +from enum import Enum import json -from dataclasses import dataclass, asdict -from typing import Any from json import JSONEncoder +from typing import Any +import uuid from optuna.storages import BaseStorage from ._system_attrs import report_preferences -from .._storage import get_study_summary _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" @@ -30,14 +30,14 @@ class Choice: class Encoder(JSONEncoder): - def default(self, o): - if isinstance(o, FeedbackMode): - return o.name - if isinstance(o, Choice): - return asdict(o) - if isinstance(o, datetime): - return o.isoformat() - return super().default(o) + def default(self, a: Any) -> Any: + if isinstance(a, FeedbackMode): + return a.name + if isinstance(a, Choice): + return asdict(a) + if isinstance(a, datetime): + return a.isoformat() + return super().default(a) def report_choice( @@ -47,7 +47,7 @@ def report_choice( preferences: list[tuple[int, int]], feedback_mode: FeedbackMode, timestamp: datetime, -): +) -> None: choice = Choice( uuid=str(uuid.uuid4()), candidate_trials=candidate_trials, diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 27e3b1b7..1e4c9346 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -27,7 +27,7 @@ const CandidateTrial: FC<{ const trialHeight = 300 const [detailShown, setDetailShown] = useState(false) - let cardComponentSx = { + const cardComponentSx = { padding: 0, position: "relative", overflow: "hidden", diff --git a/python_tests/preferential/test_history.py b/python_tests/preferential/test_history.py new file mode 100644 index 00000000..e8563b25 --- /dev/null +++ b/python_tests/preferential/test_history.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Callable + +from optuna_dashboard._serializer import serialize_preference_history +from optuna_dashboard.preferential import create_study +from optuna_dashboard.preferential._history import FeedbackMode +from optuna_dashboard.preferential._history import report_choice + +from ..storage_supplier import parametrize_storages +from ..storage_supplier import StorageSupplier + + +@parametrize_storages +def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) + study.mark_comparison_ready(trial) + + study_id = study._study._study_id + report_choice( + study_id=study_id, + storage=storage, + candidate_trials=[0, 2, 3, 4], + preferences=[(2, 0), (3, 0), (4, 0)], + feedback_mode=FeedbackMode.CHOOSE_WORST, + timestamp=datetime(2020, 1, 1, 10, 0, 1), + ) + report_choice( + study_id=study_id, + storage=storage, + candidate_trials=[0, 1, 2], + preferences=[(0, 1), (2, 1)], + feedback_mode=FeedbackMode.CHOOSE_WORST, + timestamp=datetime(2020, 1, 1, 10, 0, 0), + ) + + history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + assert len(history) == 2 + assert history[0]["candidate_trials"] == [0, 1, 2] + assert history[0]["preferences"] == [[0, 1], [2, 1]] + assert history[0]["feedback_mode"] == FeedbackMode.CHOOSE_WORST.name + assert history[0]["timestamp"] == "2020-01-01T10:00:00" + assert history[1]["candidate_trials"] == [0, 2, 3, 4] + assert history[1]["preferences"] == [[2, 0], [3, 0], [4, 0]] + assert history[1]["feedback_mode"] == FeedbackMode.CHOOSE_WORST.name + assert history[1]["timestamp"] == "2020-01-01T10:00:01" diff --git a/python_tests/test_api.py b/python_tests/test_api.py index c995c0e2..1909b461 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -136,7 +136,13 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference", "POST", - body=json.dumps({"best_trials": [0, 2], "worst_trials": [1]}), + body=json.dumps( + { + "candidate_trials": [0, 1, 2], + "preferentials": [[0, 1], [2, 1]], + "mode": "choose_worst", + } + ), content_type="application/json", ) self.assertEqual(status, 204) From 00ddfc564b7e76da653bdac7e12a72dc20d483fd Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 29 Aug 2023 17:54:50 +0900 Subject: [PATCH 03/30] fix by lint --- optuna_dashboard/preferential/_history.py | 2 ++ python_tests/preferential/test_history.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/preferential/_history.py b/optuna_dashboard/preferential/_history.py index 4170ee21..fba0c69d 100644 --- a/optuna_dashboard/preferential/_history.py +++ b/optuna_dashboard/preferential/_history.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from dataclasses import asdict from dataclasses import dataclass from datetime import datetime diff --git a/python_tests/preferential/test_history.py b/python_tests/preferential/test_history.py index e8563b25..bf80d058 100644 --- a/python_tests/preferential/test_history.py +++ b/python_tests/preferential/test_history.py @@ -42,10 +42,18 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) history = serialize_preference_history(storage.get_study_system_attrs(study_id)) assert len(history) == 2 assert history[0]["candidate_trials"] == [0, 1, 2] - assert history[0]["preferences"] == [[0, 1], [2, 1]] + assert len(history[0]["preferences"]) == 2 + for i, (best, worst) in enumerate([(0, 1), (2, 1)]): + assert len(history[0]["preferences"][i]) == 2 + assert history[0]["preferences"][i][0] == best + assert history[0]["preferences"][i][1] == worst assert history[0]["feedback_mode"] == FeedbackMode.CHOOSE_WORST.name assert history[0]["timestamp"] == "2020-01-01T10:00:00" assert history[1]["candidate_trials"] == [0, 2, 3, 4] - assert history[1]["preferences"] == [[2, 0], [3, 0], [4, 0]] + assert len(history[1]["preferences"]) == 3 + for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): + assert len(history[1]["preferences"][i]) == 2 + assert history[1]["preferences"][i][0] == best + assert history[1]["preferences"][i][1] == worst assert history[1]["feedback_mode"] == FeedbackMode.CHOOSE_WORST.name assert history[1]["timestamp"] == "2020-01-01T10:00:01" From 294c47f338e699365f0fd5648581cb6bf61716be Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 30 Aug 2023 13:51:10 +0900 Subject: [PATCH 04/30] fix by review --- optuna_dashboard/_preferential_history.py | 41 ++++++++++++ optuna_dashboard/preferential/_history.py | 65 ------------------- ...istory.py => test_preferential_history.py} | 0 3 files changed, 41 insertions(+), 65 deletions(-) create mode 100644 optuna_dashboard/_preferential_history.py delete mode 100644 optuna_dashboard/preferential/_history.py rename python_tests/{preferential/test_history.py => test_preferential_history.py} (100%) diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py new file mode 100644 index 00000000..df203f1a --- /dev/null +++ b/optuna_dashboard/_preferential_history.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from datetime import datetime +import json +from typing import TypedDict +import uuid + +from optuna.storages import BaseStorage + +from .preferential._system_attrs import report_preferences + + +_SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" + + +class Choice(TypedDict): + uuid: str + candidate_trials: list[int] + preference_uuid: str + timestamp: datetime + + +def report_choice( + study_id: int, + storage: BaseStorage, + candidate_trials: list[int], + preferences: list[tuple[int, int]], + timestamp: datetime, +) -> None: + choice: Choice = { + "uuid": str(uuid.uuid4()), + "candidate_trials": candidate_trials, + "preference_uuid": report_preferences(study_id, storage, preferences), + "timestamp": timestamp, + } + key = _SYSTEM_ATTR_PREFIX_HISTORY + choice["uuid"] + storage.set_study_system_attr( + study_id=study_id, + key=key, + value=json.dumps(choice), + ) diff --git a/optuna_dashboard/preferential/_history.py b/optuna_dashboard/preferential/_history.py deleted file mode 100644 index fba0c69d..00000000 --- a/optuna_dashboard/preferential/_history.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -from dataclasses import asdict -from dataclasses import dataclass -from datetime import datetime -from enum import Enum -import json -from json import JSONEncoder -from typing import Any -import uuid - -from optuna.storages import BaseStorage - -from ._system_attrs import report_preferences - - -_SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" - - -class FeedbackMode(Enum): - CHOOSE_WORST = 0 - AUTO = 10 - - -@dataclass -class Choice: - uuid: str - candidate_trials: list[int] - preference_uuid: str - feedback_mode: FeedbackMode - timestamp: datetime - - -class Encoder(JSONEncoder): - def default(self, a: Any) -> Any: - if isinstance(a, FeedbackMode): - return a.name - if isinstance(a, Choice): - return asdict(a) - if isinstance(a, datetime): - return a.isoformat() - return super().default(a) - - -def report_choice( - study_id: int, - storage: BaseStorage, - candidate_trials: list[int], - preferences: list[tuple[int, int]], - feedback_mode: FeedbackMode, - timestamp: datetime, -) -> None: - choice = Choice( - uuid=str(uuid.uuid4()), - candidate_trials=candidate_trials, - preference_uuid=report_preferences(study_id, storage, preferences), - feedback_mode=feedback_mode, - timestamp=timestamp, - ) - key = _SYSTEM_ATTR_PREFIX_HISTORY + choice.uuid - storage.set_study_system_attr( - study_id=study_id, - key=key, - value=json.dumps(choice, cls=Encoder), - ) diff --git a/python_tests/preferential/test_history.py b/python_tests/test_preferential_history.py similarity index 100% rename from python_tests/preferential/test_history.py rename to python_tests/test_preferential_history.py From 9d1c006f9e3aff3122c1bb7bef65db799c6e58a2 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 30 Aug 2023 14:36:58 +0900 Subject: [PATCH 05/30] fix by lint --- optuna_dashboard/_app.py | 6 ++---- optuna_dashboard/_preferential_history.py | 4 ++-- optuna_dashboard/_serializer.py | 5 +++-- python_tests/test_api.py | 1 - python_tests/test_preferential_history.py | 11 +++-------- 5 files changed, 10 insertions(+), 17 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 73e17527..c110f38f 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,6 +28,7 @@ from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preferential_history import report_choice from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -39,8 +40,6 @@ from ._storage_url import get_storage from .artifact._backend import delete_all_artifacts from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store -from .preferential._history import FeedbackMode -from .preferential._history import report_choice from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._study import get_best_trials as get_best_preferential_trials from .preferential._system_attrs import report_skip @@ -273,7 +272,6 @@ def create_app( try: candidate_trials = [int(d) for d in request.json.get("candidate_trials", [])] preferences = [(int(d[0]), int(d[1])) for d in request.json.get("preferentials", [])] - mode = FeedbackMode[request.json.get("mode", "auto").upper()] except ValueError: response.status = 400 return {"reason": "Invalid request."} @@ -281,7 +279,7 @@ def create_app( response.status = 400 # Bad request return {"reason": "You need to set best_trials and worst_trials"} - report_choice(study_id, storage, candidate_trials, preferences, mode, datetime.now()) + report_choice(study_id, storage, candidate_trials, preferences, datetime.now()) response.status = 204 return {} diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index df203f1a..d266453a 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -17,7 +17,7 @@ class Choice(TypedDict): uuid: str candidate_trials: list[int] preference_uuid: str - timestamp: datetime + timestamp: str def report_choice( @@ -31,7 +31,7 @@ def report_choice( "uuid": str(uuid.uuid4()), "candidate_trials": candidate_trials, "preference_uuid": report_preferences(study_id, storage, preferences), - "timestamp": timestamp, + "timestamp": timestamp.isoformat(), } key = _SYSTEM_ATTR_PREFIX_HISTORY + choice["uuid"] storage.set_study_system_attr( diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 741385ca..dcbccba2 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import json from typing import Any from typing import TYPE_CHECKING @@ -14,8 +15,8 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names +from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts -from .preferential._history import _SYSTEM_ATTR_PREFIX_HISTORY from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE @@ -344,5 +345,5 @@ def serialize_preference_history( _SYSTEM_ATTR_PREFIX_PREFERENCE + choice["preference_uuid"], [] ) history.append(choice) - history.sort(key=lambda c: c["timestamp"]) + history.sort(key=lambda c: datetime.fromisoformat(c["timestamp"])) return history diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 6a7fc868..8afa1021 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -140,7 +140,6 @@ class APITestCase(TestCase): { "candidate_trials": [0, 1, 2], "preferentials": [[0, 1], [2, 1]], - "mode": "choose_worst", } ), content_type="application/json", diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index bf80d058..3e1c73f4 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -3,13 +3,12 @@ from __future__ import annotations from datetime import datetime from typing import Callable +from optuna_dashboard._preferential_history import report_choice from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study -from optuna_dashboard.preferential._history import FeedbackMode -from optuna_dashboard.preferential._history import report_choice -from ..storage_supplier import parametrize_storages -from ..storage_supplier import StorageSupplier +from .storage_supplier import parametrize_storages +from .storage_supplier import StorageSupplier @parametrize_storages @@ -27,7 +26,6 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) storage=storage, candidate_trials=[0, 2, 3, 4], preferences=[(2, 0), (3, 0), (4, 0)], - feedback_mode=FeedbackMode.CHOOSE_WORST, timestamp=datetime(2020, 1, 1, 10, 0, 1), ) report_choice( @@ -35,7 +33,6 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) storage=storage, candidate_trials=[0, 1, 2], preferences=[(0, 1), (2, 1)], - feedback_mode=FeedbackMode.CHOOSE_WORST, timestamp=datetime(2020, 1, 1, 10, 0, 0), ) @@ -47,7 +44,6 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert len(history[0]["preferences"][i]) == 2 assert history[0]["preferences"][i][0] == best assert history[0]["preferences"][i][1] == worst - assert history[0]["feedback_mode"] == FeedbackMode.CHOOSE_WORST.name assert history[0]["timestamp"] == "2020-01-01T10:00:00" assert history[1]["candidate_trials"] == [0, 2, 3, 4] assert len(history[1]["preferences"]) == 3 @@ -55,5 +51,4 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert len(history[1]["preferences"][i]) == 2 assert history[1]["preferences"][i][0] == best assert history[1]["preferences"][i][1] == worst - assert history[1]["feedback_mode"] == FeedbackMode.CHOOSE_WORST.name assert history[1]["timestamp"] == "2020-01-01T10:00:01" From 9ea39e3f6ba2a43951bae0849ecaf0aa4297a1d6 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 30 Aug 2023 14:55:40 +0900 Subject: [PATCH 06/30] fix by lint --- optuna_dashboard/_preferential_history.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index d266453a..f5f14808 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -2,6 +2,7 @@ from __future__ import annotations from datetime import datetime import json +from typing import TYPE_CHECKING from typing import TypedDict import uuid @@ -12,12 +13,16 @@ from .preferential._system_attrs import report_preferences _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" - -class Choice(TypedDict): - uuid: str - candidate_trials: list[int] - preference_uuid: str - timestamp: str +if TYPE_CHECKING: + Choice = TypedDict( + "Choice", + { + "uuid": str, + "candidate_trials": list[int], + "preference_uuid": str, + "timestamp": str, + }, + ) def report_choice( From b242fd8cd9a32c235e3b6dc2aeddb0bfc77aa944 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 30 Aug 2023 16:03:47 +0900 Subject: [PATCH 07/30] fix by lint --- optuna_dashboard/_preferential_history.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index f5f14808..8e60757b 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -3,7 +3,6 @@ from __future__ import annotations from datetime import datetime import json from typing import TYPE_CHECKING -from typing import TypedDict import uuid from optuna.storages import BaseStorage @@ -14,6 +13,8 @@ from .preferential._system_attrs import report_preferences _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" if TYPE_CHECKING: + from typing import TypedDict + Choice = TypedDict( "Choice", { From 7e429266d6115b6a70f632031321f98659059bd8 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 31 Aug 2023 18:10:23 +0900 Subject: [PATCH 08/30] wip --- optuna_dashboard/_app.py | 33 +++++-- optuna_dashboard/_preferential_history.py | 112 ++++++++++++++++++---- optuna_dashboard/_serializer.py | 19 +--- optuna_dashboard/ts/apiClient.ts | 23 ++--- optuna_dashboard/ts/types/index.d.ts | 2 +- 5 files changed, 128 insertions(+), 61 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index c110f38f..87fbaa9a 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,7 +28,8 @@ from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preferential_history import report_choice +from ._preferential_history import ChooseWorstHistory +from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -263,23 +264,39 @@ def create_app( } note.save_note_with_version(storage, study_id, None, req_note_ver, req_note_body) - response.status = 204 # No content + response.status = 204 # No contenthttps://github.com/microsoft/pyright/blob/main/docs/configuration.md#reportUndefinedVariable return {} @app.post("/api/studies//preference") @json_api_view def post_preference(study_id: int) -> dict[str, Any]: try: - candidate_trials = [int(d) for d in request.json.get("candidate_trials", [])] - preferences = [(int(d[0]), int(d[1])) for d in request.json.get("preferentials", [])] + mode = request.json.get("mode", "") + candidates = [int(d) for d in request.json.get("candidates", [])] + clicked = int(request.json.get("clicked", -1)) except ValueError: response.status = 400 return {"reason": "Invalid request."} - if len(preferences) == 0: - response.status = 400 # Bad request - return {"reason": "You need to set best_trials and worst_trials"} - report_choice(study_id, storage, candidate_trials, preferences, datetime.now()) + if clicked == -1: + response.status = 400 + return {"reason": "`clicked` should be specified."} + + if mode == "ChooseWorst": + history = ChooseWorstHistory.create( + study_id, + storage, + { + "mode": mode, + "candidates": candidates, + "clicked": clicked, + }, + ) + else: + response.status = 400 + return {"reason": "`mode` should be 'ChooseWorst'."} + + report_history(study_id, storage, history) response.status = 204 return {} diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 8e60757b..52f10c3b 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -1,12 +1,16 @@ from __future__ import annotations +from dataclasses import dataclass from datetime import datetime import json +from typing import Any +from typing import Literal from typing import TYPE_CHECKING import uuid from optuna.storages import BaseStorage +from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE from .preferential._system_attrs import report_preferences @@ -15,33 +19,105 @@ _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" if TYPE_CHECKING: from typing import TypedDict - Choice = TypedDict( - "Choice", + NewChooseWorstHistoryJSON = TypedDict( + "NewChooseWorstHistoryJSON", { - "uuid": str, - "candidate_trials": list[int], - "preference_uuid": str, - "timestamp": str, + "mode": Literal["ChooseWorst"], + "candidates": list[int], + "clicked": int, }, ) + NewHistoryJSON = NewChooseWorstHistoryJSON -def report_choice( + +@dataclass(frozen=True) +class ChooseWorstHistory: + mode: Literal["ChooseWorst"] + uuid: str + preference_uuid: str # making it possible to remove the preference + timestamp: datetime + candidates: list[int] # a list of trial number + clicked: int # The worst trial number in the candidates. + + def to_dict(self) -> dict[str, Any]: + return { + "mode": self.mode, + "uuid": self.uuid, + "preference_uuid": self.preference_uuid, + "timestamp": self.timestamp.isoformat(), + "candidates": self.candidates, + "clicked": self.clicked, + } + + def to_response(self, system_attrs: dict[str, Any]) -> dict[str, Any]: + return { + "uuid": self.uuid, + "preferences": system_attrs.get( + _SYSTEM_ATTR_PREFIX_PREFERENCE + self.preference_uuid, [] + ), + "timestamp": self.timestamp.isoformat(), + "candidates": self.candidates, + "clicked": self.clicked, + } + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> ChooseWorstHistory: + return ChooseWorstHistory( + mode="ChooseWorst", + uuid=d["uuid"], + preference_uuid=d["preference_uuid"], + timestamp=datetime.fromisoformat(d["timestamp"]), + candidates=d["candidates"], + clicked=d["clicked"], + ) + + @classmethod + def create(cls, study_id: int, storage: BaseStorage, d: NewHistoryJSON) -> ChooseWorstHistory: + preferences = [(best, d["clicked"]) for best in d["candidates"] if best != d["clicked"]] + preference_uuid = report_preferences( + study_id=study_id, + storage=storage, + preferences=preferences, + ) + return ChooseWorstHistory( + mode="ChooseWorst", + uuid=str(uuid.uuid4()), + preference_uuid=preference_uuid, + timestamp=datetime.now(), + candidates=d["candidates"], + clicked=d["clicked"], + ) + + +History = ChooseWorstHistory + + +def report_history( study_id: int, storage: BaseStorage, - candidate_trials: list[int], - preferences: list[tuple[int, int]], - timestamp: datetime, + history: History, ) -> None: - choice: Choice = { - "uuid": str(uuid.uuid4()), - "candidate_trials": candidate_trials, - "preference_uuid": report_preferences(study_id, storage, preferences), - "timestamp": timestamp.isoformat(), - } - key = _SYSTEM_ATTR_PREFIX_HISTORY + choice["uuid"] + key = _SYSTEM_ATTR_PREFIX_HISTORY + history.uuid storage.set_study_system_attr( study_id=study_id, key=key, - value=json.dumps(choice), + value=json.dumps(history.to_dict()), ) + + +def serialize_preference_history( + system_attrs: dict[str, Any], +) -> list[dict[str, Any]]: + histories: list[History] = [] + for k, v in system_attrs.items(): + if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): + continue + choice: dict[str, Any] = json.loads(v) + if choice["mode"] == "ChooseWorst": + histories.append(ChooseWorstHistory.from_dict(choice)) + else: + assert False + + histories.sort(key=lambda c: c.timestamp) + return [history.to_response(system_attrs) for history in histories] diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index dcbccba2..52ca620c 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,10 +15,9 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names -from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY +from ._preferential_history import serialize_preference_history from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY -from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE if TYPE_CHECKING: @@ -331,19 +330,3 @@ def serialize_search_space( } ) return serialized - - -def serialize_preference_history( - system_attrs: dict[str, Any], -) -> list[dict[str, Any]]: - history: list[dict[str, Any]] = [] - for k, v in system_attrs.items(): - if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): - continue - choice: dict[str, Any] = json.loads(v) - choice["preferences"] = system_attrs.get( - _SYSTEM_ATTR_PREFIX_PREFERENCE + choice["preference_uuid"], [] - ) - history.append(choice) - history.sort(key=lambda c: datetime.fromisoformat(c["timestamp"])) - return history diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 69e43e74..d3cddc89 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -57,9 +57,9 @@ const convertTrialResponse = (res: TrialResponse): Trial => { interface PreferenceChoiceResponce { uuid: string - candidate_trials: number[] + candidates: number[] preferences: number[][] - feedback_mode: PreferenceFeedbackMode + mode: PreferenceFeedbackMode timestamp: string } @@ -68,9 +68,9 @@ const convertPreferenceChoice = ( ): PreferenceChoice => { return { uuid: res.uuid, - candidate_trials: res.candidate_trials, + candidate_trials: res.candidates, preferences: res.preferences, - feedback_mode: res.feedback_mode, + feedback_mode: res.mode, timestamp: new Date(res.timestamp), } } @@ -343,18 +343,9 @@ export const reportPreferenceAPI = ( ): Promise => { return axiosInstance .post(`/api/studies/${studyId}/preference`, { - candidate_trials: best_trials.concat(worst_trials), - preferentials: best_trials.reduce( - (prev, best_trial_id) => - prev.concat( - worst_trials.map((worst_trial_id) => [ - best_trial_id, - worst_trial_id, - ]) - ), - [] - ), - mode: "choose_worst", + candidates: best_trials.concat(worst_trials), + clicked: worst_trials[0], + mode: "ChooseWorst", }) .then(() => { return diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 9e8e408f..358cb17c 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -12,7 +12,7 @@ type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type TrialStateFinished = "Complete" | "Fail" | "Pruned" type StudyDirection = "maximize" | "minimize" | "not_set" -type PreferenceFeedbackMode = "choose_worst" | "auto" +type PreferenceFeedbackMode = "ChooseWorst" type FloatDistribution = { type: "FloatDistribution" From b09e223e17710db2292ba9845bc291b0a971f1e0 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 1 Sep 2023 12:03:45 +0900 Subject: [PATCH 09/30] modify API --- optuna_dashboard/_app.py | 24 ++--- optuna_dashboard/_preferential_history.py | 92 +++++++++---------- optuna_dashboard/ts/action.ts | 6 +- optuna_dashboard/ts/apiClient.ts | 14 +-- .../ts/components/PreferenceHistory.tsx | 10 +- .../ts/components/PreferentialTrials.tsx | 11 +-- optuna_dashboard/ts/types/index.d.ts | 4 +- python_tests/test_api.py | 5 +- python_tests/test_preferential_history.py | 62 +++++++------ 9 files changed, 109 insertions(+), 119 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 87fbaa9a..60eccd8d 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,7 +28,6 @@ from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preferential_history import ChooseWorstHistory from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail @@ -281,22 +280,19 @@ def create_app( if clicked == -1: response.status = 400 return {"reason": "`clicked` should be specified."} - - if mode == "ChooseWorst": - history = ChooseWorstHistory.create( - study_id, - storage, - { - "mode": mode, - "candidates": candidates, - "clicked": clicked, - }, - ) - else: + if mode != "ChooseWorst": response.status = 400 return {"reason": "`mode` should be 'ChooseWorst'."} - report_history(study_id, storage, history) + report_history( + study_id, + storage, + { + "mode": mode, + "candidates": candidates, + "clicked": clicked, + }, + ) response.status = 204 return {} diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 52f10c3b..5c930932 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -10,7 +10,6 @@ import uuid from optuna.storages import BaseStorage -from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE from .preferential._system_attrs import report_preferences @@ -19,8 +18,8 @@ _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" if TYPE_CHECKING: from typing import TypedDict - NewChooseWorstHistoryJSON = TypedDict( - "NewChooseWorstHistoryJSON", + NewHistoryJSON = TypedDict( + "NewHistoryJSON", { "mode": Literal["ChooseWorst"], "candidates": list[int], @@ -28,8 +27,6 @@ if TYPE_CHECKING: }, ) - NewHistoryJSON = NewChooseWorstHistoryJSON - @dataclass(frozen=True) class ChooseWorstHistory: @@ -50,45 +47,6 @@ class ChooseWorstHistory: "clicked": self.clicked, } - def to_response(self, system_attrs: dict[str, Any]) -> dict[str, Any]: - return { - "uuid": self.uuid, - "preferences": system_attrs.get( - _SYSTEM_ATTR_PREFIX_PREFERENCE + self.preference_uuid, [] - ), - "timestamp": self.timestamp.isoformat(), - "candidates": self.candidates, - "clicked": self.clicked, - } - - @classmethod - def from_dict(cls, d: dict[str, Any]) -> ChooseWorstHistory: - return ChooseWorstHistory( - mode="ChooseWorst", - uuid=d["uuid"], - preference_uuid=d["preference_uuid"], - timestamp=datetime.fromisoformat(d["timestamp"]), - candidates=d["candidates"], - clicked=d["clicked"], - ) - - @classmethod - def create(cls, study_id: int, storage: BaseStorage, d: NewHistoryJSON) -> ChooseWorstHistory: - preferences = [(best, d["clicked"]) for best in d["candidates"] if best != d["clicked"]] - preference_uuid = report_preferences( - study_id=study_id, - storage=storage, - preferences=preferences, - ) - return ChooseWorstHistory( - mode="ChooseWorst", - uuid=str(uuid.uuid4()), - preference_uuid=preference_uuid, - timestamp=datetime.now(), - candidates=d["candidates"], - clicked=d["clicked"], - ) - History = ChooseWorstHistory @@ -96,9 +54,36 @@ History = ChooseWorstHistory def report_history( study_id: int, storage: BaseStorage, - history: History, + input_data: NewHistoryJSON, ) -> None: - key = _SYSTEM_ATTR_PREFIX_HISTORY + history.uuid + preferences = [] + if input_data["mode"] == "ChooseWorst": + preferences = [ + (best, input_data["clicked"]) + for best in input_data["candidates"] + if best != input_data["clicked"] + ] + else: + assert False, f"Unknown mode: {input_data['mode']}" + + preference_uuid = report_preferences( + study_id=study_id, + storage=storage, + preferences=preferences, + ) + history_uuid = str(uuid.uuid4()) + + if input_data["mode"] == "ChooseWorst": + history = ChooseWorstHistory( + mode="ChooseWorst", + uuid=history_uuid, + preference_uuid=preference_uuid, + timestamp=datetime.now(), + candidates=input_data["candidates"], + clicked=input_data["clicked"], + ) + + key = _SYSTEM_ATTR_PREFIX_HISTORY + history_uuid storage.set_study_system_attr( study_id=study_id, key=key, @@ -115,9 +100,18 @@ def serialize_preference_history( continue choice: dict[str, Any] = json.loads(v) if choice["mode"] == "ChooseWorst": - histories.append(ChooseWorstHistory.from_dict(choice)) + histories.append( + ChooseWorstHistory( + mode="ChooseWorst", + uuid=choice["uuid"], + preference_uuid=choice["preference_uuid"], + timestamp=datetime.fromisoformat(choice["timestamp"]), + candidates=choice["candidates"], + clicked=choice["clicked"], + ) + ) else: - assert False + assert False, f"Unknown mode: {choice['mode']}" histories.sort(key=lambda c: c.timestamp) - return [history.to_response(system_attrs) for history in histories] + return [history.to_dict() for history in histories] diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index 017751fa..e1896178 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -587,10 +587,10 @@ export const actionCreator = () => { const updatePreference = ( study_id: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ) => { - reportPreferenceAPI(study_id, best_trials, worst_trials).catch((err) => { + reportPreferenceAPI(study_id, candidates, clicked).catch((err) => { const reason = err.response?.data.reason enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { variant: "error", diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index d3cddc89..9c72f739 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -58,7 +58,7 @@ const convertTrialResponse = (res: TrialResponse): Trial => { interface PreferenceChoiceResponce { uuid: string candidates: number[] - preferences: number[][] + clicked: number mode: PreferenceFeedbackMode timestamp: string } @@ -68,8 +68,8 @@ const convertPreferenceChoice = ( ): PreferenceChoice => { return { uuid: res.uuid, - candidate_trials: res.candidates, - preferences: res.preferences, + candidates: res.candidates, + clicked: res.clicked, feedback_mode: res.mode, timestamp: new Date(res.timestamp), } @@ -338,13 +338,13 @@ export const getParamImportances = ( export const reportPreferenceAPI = ( studyId: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ): Promise => { return axiosInstance .post(`/api/studies/${studyId}/preference`, { - candidates: best_trials.concat(worst_trials), - clicked: worst_trials[0], + candidates: candidates, + clicked: clicked, mode: "ChooseWorst", }) .then(() => { diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 1e4c9346..e95a4cb1 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -138,13 +138,7 @@ const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ trials, }) => { const theme = useTheme() - const worst_trials = new Set(choice.preferences.map((pair) => pair[1])) - console.log( - worst_trials, - choice.candidate_trials, - choice.preferences, - choice.preferences.map((pair) => pair[1]) - ) + const worst_trials = new Set([choice.clicked]) return ( @@ -163,7 +157,7 @@ const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ flexDirection: "row", }} > - {choice.candidate_trials.map((trial_num, index) => ( + {choice.candidates.map((trial_num, index) => ( void -}> = ({ trial, studyDetail, hideTrial }) => { +}> = ({ trial, candidates, hideTrial }) => { const theme = useTheme() const action = actionCreator() const trialWidth = 500 @@ -80,10 +80,7 @@ const PreferentialTrial: FC<{ aria-label="trial-button" onClick={() => { hideTrial() - const best_trials = studyDetail.best_trials - .map((t) => t.number) - .filter((t) => t !== trial.number) - action.updatePreference(trial.study_id, best_trials, [trial.number]) + action.updatePreference(trial.study_id, candidates, trial.number) }} sx={{ padding: 0, @@ -243,7 +240,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ trial.number === t)} - studyDetail={studyDetail} + candidates={displayTrials.numbers.filter((n) => n !== -1)} hideTrial={() => { hideTrial(t) }} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 358cb17c..da90ad1d 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -211,8 +211,8 @@ type StudyParamImportance = { type PreferenceChoice = { uuid: string - candidate_trials: number[] - preferences: number[][] + candidates: number[] + clicked: number feedback_mode: PreferenceFeedbackMode timestamp: Date } diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 8afa1021..7d732e37 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -138,8 +138,9 @@ class APITestCase(TestCase): "POST", body=json.dumps( { - "candidate_trials": [0, 1, 2], - "preferentials": [[0, 1], [2, 1]], + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 1, } ), content_type="application/json", diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 3e1c73f4..3448abc3 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -3,7 +3,8 @@ from __future__ import annotations from datetime import datetime from typing import Callable -from optuna_dashboard._preferential_history import report_choice +from optuna_dashboard._preferential_history import report_history +from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study @@ -21,34 +22,41 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) study.mark_comparison_ready(trial) study_id = study._study._study_id - report_choice( - study_id=study_id, - storage=storage, - candidate_trials=[0, 2, 3, 4], - preferences=[(2, 0), (3, 0), (4, 0)], - timestamp=datetime(2020, 1, 1, 10, 0, 1), - ) - report_choice( - study_id=study_id, - storage=storage, - candidate_trials=[0, 1, 2], - preferences=[(0, 1), (2, 1)], - timestamp=datetime(2020, 1, 1, 10, 0, 0), - ) + report_history( + study_id=study_id, + storage=storage, + input_data={ + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + }, + ) + report_history( + study_id=study_id, + storage=storage, + input_data={ + "mode": "ChooseWorst", + "candidates": [0, 2, 3, 4], + "clicked": 0, + }, + ) history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + sys_attrs = storage.get_study_system_attrs(study_id) assert len(history) == 2 - assert history[0]["candidate_trials"] == [0, 1, 2] - assert len(history[0]["preferences"]) == 2 + assert history[0]["candidates"] == [0, 1, 2] + assert history[0]["clicked"] == 1 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_uuid"]] + assert len(preferences) == 2 for i, (best, worst) in enumerate([(0, 1), (2, 1)]): - assert len(history[0]["preferences"][i]) == 2 - assert history[0]["preferences"][i][0] == best - assert history[0]["preferences"][i][1] == worst - assert history[0]["timestamp"] == "2020-01-01T10:00:00" - assert history[1]["candidate_trials"] == [0, 2, 3, 4] - assert len(history[1]["preferences"]) == 3 + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst + assert history[1]["candidates"] == [0, 2, 3, 4] + assert history[1]["clicked"] == 0 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_uuid"]] + assert len(preferences) == 3 for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): - assert len(history[1]["preferences"][i]) == 2 - assert history[1]["preferences"][i][0] == best - assert history[1]["preferences"][i][1] == worst - assert history[1]["timestamp"] == "2020-01-01T10:00:01" + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst From 3e13de1b76de6c6cc38c1171beb3d8ec7e869ea9 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 1 Sep 2023 12:12:30 +0900 Subject: [PATCH 10/30] fix by lint --- optuna_dashboard/_app.py | 3 +-- optuna_dashboard/_serializer.py | 1 - optuna_dashboard/ts/components/App.tsx | 2 +- optuna_dashboard/ts/components/AppDrawer.tsx | 2 +- python_tests/test_preferential_history.py | 3 +-- 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 60eccd8d..f582d5e5 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -1,6 +1,5 @@ from __future__ import annotations -from datetime import datetime import functools import logging import os @@ -263,7 +262,7 @@ def create_app( } note.save_note_with_version(storage, study_id, None, req_note_ver, req_note_body) - response.status = 204 # No contenthttps://github.com/microsoft/pyright/blob/main/docs/configuration.md#reportUndefinedVariable + response.status = 204 # No content return {} @app.post("/api/studies//preference") diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 52ca620c..19ceb19b 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -1,6 +1,5 @@ from __future__ import annotations -from datetime import datetime import json from typing import Any from typing import TYPE_CHECKING diff --git a/optuna_dashboard/ts/components/App.tsx b/optuna_dashboard/ts/components/App.tsx index e904ae30..8adf8895 100644 --- a/optuna_dashboard/ts/components/App.tsx +++ b/optuna_dashboard/ts/components/App.tsx @@ -97,7 +97,7 @@ export const App: FC = () => { } /> diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 3448abc3..64591783 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -1,12 +1,11 @@ from __future__ import annotations -from datetime import datetime from typing import Callable from optuna_dashboard._preferential_history import report_history -from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study +from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE from .storage_supplier import parametrize_storages from .storage_supplier import StorageSupplier From 22fdc50102100a6cce39eaf6a48a2c0177a6c505 Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 1 Sep 2023 13:21:42 +0900 Subject: [PATCH 11/30] Bump the version up to v0.13.0b1 --- optuna_dashboard/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 3d363cf4..0ab0cd89 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -15,4 +15,4 @@ from ._note import get_note # noqa from ._note import save_note # noqa -__version__ = "0.12.0" +__version__ = "0.13.0b1" From 7a97d3db5ad6c464459c6ea3679b02c1eea15fd8 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 1 Sep 2023 18:39:16 +0900 Subject: [PATCH 12/30] fix by lint --- optuna_dashboard/_preferential_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 5c930932..f4660310 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -4,7 +4,6 @@ from dataclasses import dataclass from datetime import datetime import json from typing import Any -from typing import Literal from typing import TYPE_CHECKING import uuid @@ -17,6 +16,7 @@ _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" if TYPE_CHECKING: from typing import TypedDict + from typing import Literal NewHistoryJSON = TypedDict( "NewHistoryJSON", From 1d7d5a7252222b9d6c5d975de36a0826ba3f06f2 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 4 Sep 2023 12:15:29 +0900 Subject: [PATCH 13/30] fix by review --- optuna_dashboard/_app.py | 11 +++++---- optuna_dashboard/_preferential_history.py | 22 +++++++++++++---- optuna_dashboard/ts/components/AppDrawer.tsx | 2 +- .../ts/components/PreferenceHistory.tsx | 5 ++-- python_tests/test_api.py | 24 +++++++++++++++++++ 5 files changed, 52 insertions(+), 12 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index f582d5e5..49a60957 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -27,6 +27,7 @@ from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preferential_history import cast_feedback_mode from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail @@ -269,19 +270,19 @@ def create_app( @json_api_view def post_preference(study_id: int) -> dict[str, Any]: try: - mode = request.json.get("mode", "") + mode = cast_feedback_mode(request.json.get("mode", "")) candidates = [int(d) for d in request.json.get("candidates", [])] clicked = int(request.json.get("clicked", -1)) - except ValueError: + except Exception: response.status = 400 return {"reason": "Invalid request."} if clicked == -1: response.status = 400 return {"reason": "`clicked` should be specified."} - if mode != "ChooseWorst": - response.status = 400 - return {"reason": "`mode` should be 'ChooseWorst'."} + # if mode != "ChooseWorst": + # response.status = 400 + # return {"reason": "`mode` should be 'ChooseWorst'."} report_history( study_id, diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index f4660310..185d3de0 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -3,31 +3,47 @@ from __future__ import annotations from dataclasses import dataclass from datetime import datetime import json +import sys from typing import Any from typing import TYPE_CHECKING import uuid from optuna.storages import BaseStorage +from typeguard import typechecked from .preferential._system_attrs import report_preferences _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" +if TYPE_CHECKING or (3, 8, 0) <= sys.version_info: + from typing import Literal +else: + from typing_extensions import Literal +FeedbackMode = Literal["ChooseWorst"] if TYPE_CHECKING: from typing import TypedDict - from typing import Literal NewHistoryJSON = TypedDict( "NewHistoryJSON", { - "mode": Literal["ChooseWorst"], + "mode": FeedbackMode, "candidates": list[int], "clicked": int, }, ) +@typechecked +def check_feedback_mode(mode: FeedbackMode) -> None: + pass + + +def cast_feedback_mode(mode: str) -> FeedbackMode: + check_feedback_mode(mode) # type: ignore + return mode # type: ignore + + @dataclass(frozen=True) class ChooseWorstHistory: mode: Literal["ChooseWorst"] @@ -110,8 +126,6 @@ def serialize_preference_history( clicked=choice["clicked"], ) ) - else: - assert False, f"Unknown mode: {choice['mode']}" histories.sort(key=lambda c: c.timestamp) return [history.to_dict() for history in histories] diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 184ec606..446b362e 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -221,7 +221,7 @@ export const AppDrawer: FC<{ component={Link} to={`${URL_PREFIX}/studies/${studyId}/preference-history`} sx={styleListItemButton} - selected={page === "analytics"} + selected={page === "preferenceHistory"} > diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index e95a4cb1..53cedfbb 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -11,10 +11,11 @@ import ClearIcon from "@mui/icons-material/Clear" import IconButton from "@mui/material/IconButton" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import Modal from "@mui/material/Modal" +import { red } from "@mui/material/colors" import { TrialListDetail } from "./TrialList" import { MarkdownRenderer } from "./Note" -import { red } from "@mui/material/colors" +import { formatDate } from "../dateUtil" type TrialType = "worst" | "none" @@ -149,7 +150,7 @@ const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ fontWeight: theme.typography.fontWeightLight, }} > - {choice.timestamp.toISOString()} + {formatDate(choice.timestamp)} None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage) + for _ in range(3): + trial = study.ask() + study.mark_comparison_ready(trial) + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference", + "POST", + body=json.dumps( + { + "mode": "ChoseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 400) + def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage) From afd44db711053eb65e2204875967ac92822b0046 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 4 Sep 2023 16:53:17 +0900 Subject: [PATCH 14/30] remove dependency --- optuna_dashboard/_app.py | 9 +++--- optuna_dashboard/_preferential_history.py | 37 ++++++++--------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 49a60957..b5dce8d0 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -27,7 +27,6 @@ from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preferential_history import cast_feedback_mode from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail @@ -270,7 +269,7 @@ def create_app( @json_api_view def post_preference(study_id: int) -> dict[str, Any]: try: - mode = cast_feedback_mode(request.json.get("mode", "")) + mode = request.json.get("mode", "") candidates = [int(d) for d in request.json.get("candidates", [])] clicked = int(request.json.get("clicked", -1)) except Exception: @@ -280,9 +279,9 @@ def create_app( if clicked == -1: response.status = 400 return {"reason": "`clicked` should be specified."} - # if mode != "ChooseWorst": - # response.status = 400 - # return {"reason": "`mode` should be 'ChooseWorst'."} + if mode != "ChooseWorst": + response.status = 400 + return {"reason": "`mode` should be 'ChooseWorst'."} report_history( study_id, diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 185d3de0..7ff588f5 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -3,45 +3,32 @@ from __future__ import annotations from dataclasses import dataclass from datetime import datetime import json -import sys from typing import Any from typing import TYPE_CHECKING import uuid from optuna.storages import BaseStorage -from typeguard import typechecked from .preferential._system_attrs import report_preferences _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" -if TYPE_CHECKING or (3, 8, 0) <= sys.version_info: - from typing import Literal -else: - from typing_extensions import Literal -FeedbackMode = Literal["ChooseWorst"] if TYPE_CHECKING: - from typing import TypedDict + from typing import Literal - NewHistoryJSON = TypedDict( - "NewHistoryJSON", - { - "mode": FeedbackMode, - "candidates": list[int], - "clicked": int, - }, - ) + FeedbackMode = Literal["ChooseWorst"] + if TYPE_CHECKING: + from typing import TypedDict - -@typechecked -def check_feedback_mode(mode: FeedbackMode) -> None: - pass - - -def cast_feedback_mode(mode: str) -> FeedbackMode: - check_feedback_mode(mode) # type: ignore - return mode # type: ignore + NewHistoryJSON = TypedDict( + "NewHistoryJSON", + { + "mode": FeedbackMode, + "candidates": list[int], + "clicked": int, + }, + ) @dataclass(frozen=True) From 14b253e29fd33ff35303d0b4fec788d7c1b6ad16 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 5 Sep 2023 11:15:42 +0900 Subject: [PATCH 15/30] fix by review --- optuna_dashboard/_app.py | 9 +++++++-- optuna_dashboard/ts/apiClient.ts | 12 ++++++------ .../ts/components/PreferenceHistory.tsx | 17 +++++++++++------ optuna_dashboard/ts/types/index.d.ts | 4 ++-- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index b5dce8d0..cbf257a8 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -272,9 +272,14 @@ def create_app( mode = request.json.get("mode", "") candidates = [int(d) for d in request.json.get("candidates", [])] clicked = int(request.json.get("clicked", -1)) - except Exception: + except ValueError: response.status = 400 - return {"reason": "Invalid request."} + return { + "reason": ( + "`candidates` should be an array of integers and " + "`clicked` should be an integer." + ) + } if clicked == -1: response.status = 400 diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 9c72f739..2c202f68 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -55,7 +55,7 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } -interface PreferenceChoiceResponce { +interface PreferenceHistoryResponce { uuid: string candidates: number[] clicked: number @@ -63,9 +63,9 @@ interface PreferenceChoiceResponce { timestamp: string } -const convertPreferenceChoice = ( - res: PreferenceChoiceResponce -): PreferenceChoice => { +const convertPreferenceHistory = ( + res: PreferenceHistoryResponce +): PreferenceHistory => { return { uuid: res.uuid, candidates: res.candidates, @@ -90,7 +90,7 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets - preference_history?: PreferenceChoiceResponce[] + preference_history?: PreferenceHistoryResponce[] } export const getStudyDetailAPI = ( @@ -127,7 +127,7 @@ export const getStudyDetailAPI = ( form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, preference_history: res.data.preference_history?.map( - convertPreferenceChoice + convertPreferenceHistory ), } }) diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 53cedfbb..f21fd326 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -24,7 +24,7 @@ const CandidateTrial: FC<{ type: TrialType }> = ({ trial, type }) => { const theme = useTheme() - const trialWidth = 500 + const trialWidth = 300 const trialHeight = 300 const [detailShown, setDetailShown] = useState(false) @@ -134,7 +134,7 @@ const CandidateTrial: FC<{ ) } -const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ +const ChoiceTrials: FC<{ choice: PreferenceHistory; trials: Trial[] }> = ({ choice, trials, }) => { @@ -142,11 +142,14 @@ const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ const worst_trials = new Set([choice.clicked]) return ( - + @@ -156,6 +159,7 @@ const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ sx={{ display: "flex", flexDirection: "row", + flexWrap: "wrap", }} > {choice.candidates.map((trial_num, index) => ( @@ -181,8 +185,9 @@ export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({ return null } const theme = useTheme() + const preference_histories = [...studyDetail.preference_history] - if (studyDetail.preference_history.length === 0) { + if (preference_histories.length === 0) { return ( = ({ padding={theme.spacing(2)} sx={{ display: "flex", flexDirection: "column" }} > - {studyDetail.preference_history.map((choice) => ( + {preference_histories.reverse().map((choice) => ( Date: Tue, 5 Sep 2023 13:17:41 +0900 Subject: [PATCH 16/30] fix by review --- optuna_dashboard/_preferential_history.py | 22 +++++++++++----------- optuna_dashboard/ts/apiClient.ts | 6 ++++-- optuna_dashboard/ts/types/index.d.ts | 3 ++- python_tests/test_preferential_history.py | 4 ++-- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 7ff588f5..00375e56 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -34,8 +34,8 @@ if TYPE_CHECKING: @dataclass(frozen=True) class ChooseWorstHistory: mode: Literal["ChooseWorst"] - uuid: str - preference_uuid: str # making it possible to remove the preference + id: str + preference_id: str # making it possible to remove the preference timestamp: datetime candidates: list[int] # a list of trial number clicked: int # The worst trial number in the candidates. @@ -43,8 +43,8 @@ class ChooseWorstHistory: def to_dict(self) -> dict[str, Any]: return { "mode": self.mode, - "uuid": self.uuid, - "preference_uuid": self.preference_uuid, + "id": self.id, + "preference_id": self.preference_id, "timestamp": self.timestamp.isoformat(), "candidates": self.candidates, "clicked": self.clicked, @@ -69,24 +69,24 @@ def report_history( else: assert False, f"Unknown mode: {input_data['mode']}" - preference_uuid = report_preferences( + preference_id = report_preferences( study_id=study_id, storage=storage, preferences=preferences, ) - history_uuid = str(uuid.uuid4()) + history_id = str(uuid.uuid4()) if input_data["mode"] == "ChooseWorst": history = ChooseWorstHistory( mode="ChooseWorst", - uuid=history_uuid, - preference_uuid=preference_uuid, + id=history_id, + preference_id=preference_id, timestamp=datetime.now(), candidates=input_data["candidates"], clicked=input_data["clicked"], ) - key = _SYSTEM_ATTR_PREFIX_HISTORY + history_uuid + key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id storage.set_study_system_attr( study_id=study_id, key=key, @@ -106,8 +106,8 @@ def serialize_preference_history( histories.append( ChooseWorstHistory( mode="ChooseWorst", - uuid=choice["uuid"], - preference_uuid=choice["preference_uuid"], + id=choice["id"], + preference_id=choice["preference_id"], timestamp=datetime.fromisoformat(choice["timestamp"]), candidates=choice["candidates"], clicked=choice["clicked"], diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 2c202f68..e23fc2ff 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -56,7 +56,8 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } interface PreferenceHistoryResponce { - uuid: string + id: string + preference_id: string candidates: number[] clicked: number mode: PreferenceFeedbackMode @@ -67,7 +68,8 @@ const convertPreferenceHistory = ( res: PreferenceHistoryResponce ): PreferenceHistory => { return { - uuid: res.uuid, + id: res.id, + preference_id: res.preference_id, candidates: res.candidates, clicked: res.clicked, feedback_mode: res.mode, diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index dbe35d75..646d64cf 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -210,7 +210,8 @@ type StudyParamImportance = { } type PreferenceHistory = { - uuid: string + id: string + preference_id: string candidates: number[] clicked: number feedback_mode: PreferenceFeedbackMode diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 64591783..b23dee9e 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -45,7 +45,7 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert len(history) == 2 assert history[0]["candidates"] == [0, 1, 2] assert history[0]["clicked"] == 1 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_uuid"]] + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]] assert len(preferences) == 2 for i, (best, worst) in enumerate([(0, 1), (2, 1)]): assert len(preferences[i]) == 2 @@ -53,7 +53,7 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert preferences[i][1] == worst assert history[1]["candidates"] == [0, 2, 3, 4] assert history[1]["clicked"] == 0 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_uuid"]] + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]] assert len(preferences) == 3 for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): assert len(preferences[i]) == 2 From 2ed588765210f991f59e57c575fcbdc840ccedcb Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 5 Sep 2023 18:14:42 +0900 Subject: [PATCH 17/30] fix by review --- optuna_dashboard/_app.py | 11 +- optuna_dashboard/_preferential_history.py | 110 ++++++------------ optuna_dashboard/_serializer.py | 28 ++++- .../preferential/_system_attrs.py | 6 +- .../ts/components/PreferenceHistory.tsx | 2 +- python_tests/test_preferential_history.py | 21 ++-- 6 files changed, 86 insertions(+), 92 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index cbf257a8..5f433072 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -27,6 +27,7 @@ from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preferential_history import NewHistory from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail @@ -291,11 +292,11 @@ def create_app( report_history( study_id, storage, - { - "mode": mode, - "candidates": candidates, - "clicked": clicked, - }, + NewHistory( + mode=mode, + candidates=candidates, + clicked=clicked, + ), ) response.status = 204 diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 00375e56..a246c166 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -16,58 +16,48 @@ _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" if TYPE_CHECKING: from typing import Literal + from typing import TypedDict FeedbackMode = Literal["ChooseWorst"] - if TYPE_CHECKING: - from typing import TypedDict - - NewHistoryJSON = TypedDict( - "NewHistoryJSON", - { - "mode": FeedbackMode, - "candidates": list[int], - "clicked": int, - }, - ) + ChooseWorstHistory = TypedDict( + "ChooseWorstHistory", + { + "mode": FeedbackMode, + "id": str, + "preference_id": str, + "timestamp": str, + "candidates": list[int], + "clicked": int, + }, + ) + History = ChooseWorstHistory +else: + ChooseWorstHistory = Any + History = Any -@dataclass(frozen=True) -class ChooseWorstHistory: - mode: Literal["ChooseWorst"] - id: str - preference_id: str # making it possible to remove the preference - timestamp: datetime - candidates: list[int] # a list of trial number - clicked: int # The worst trial number in the candidates. - - def to_dict(self) -> dict[str, Any]: - return { - "mode": self.mode, - "id": self.id, - "preference_id": self.preference_id, - "timestamp": self.timestamp.isoformat(), - "candidates": self.candidates, - "clicked": self.clicked, - } - - -History = ChooseWorstHistory +@dataclass +class NewHistory: + mode: FeedbackMode + candidates: list[int] + clicked: int def report_history( study_id: int, storage: BaseStorage, - input_data: NewHistoryJSON, + input_data: NewHistory, ) -> None: preferences = [] - if input_data["mode"] == "ChooseWorst": + # TODO(moririn): Use TypeGuard after adding other history types. + if input_data.mode == "ChooseWorst": preferences = [ - (best, input_data["clicked"]) - for best in input_data["candidates"] - if best != input_data["clicked"] + (best, input_data.clicked) + for best in input_data.candidates + if best != input_data.clicked ] else: - assert False, f"Unknown mode: {input_data['mode']}" + assert False, f"Unknown data: {input_data}" preference_id = report_preferences( study_id=study_id, @@ -76,43 +66,19 @@ def report_history( ) history_id = str(uuid.uuid4()) - if input_data["mode"] == "ChooseWorst": - history = ChooseWorstHistory( - mode="ChooseWorst", - id=history_id, - preference_id=preference_id, - timestamp=datetime.now(), - candidates=input_data["candidates"], - clicked=input_data["clicked"], - ) + if input_data.mode == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": history_id, + "preference_id": preference_id, + "timestamp": datetime.now().isoformat(), + "candidates": input_data.candidates, + "clicked": input_data.clicked, + } key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id storage.set_study_system_attr( study_id=study_id, key=key, - value=json.dumps(history.to_dict()), + value=json.dumps(history), ) - - -def serialize_preference_history( - system_attrs: dict[str, Any], -) -> list[dict[str, Any]]: - histories: list[History] = [] - for k, v in system_attrs.items(): - if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): - continue - choice: dict[str, Any] = json.loads(v) - if choice["mode"] == "ChooseWorst": - histories.append( - ChooseWorstHistory( - mode="ChooseWorst", - id=choice["id"], - preference_id=choice["preference_id"], - timestamp=datetime.fromisoformat(choice["timestamp"]), - candidates=choice["candidates"], - clicked=choice["clicked"], - ) - ) - - histories.sort(key=lambda c: c.timestamp) - return [history.to_dict() for history in histories] diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 19ceb19b..a0999b1f 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import json from typing import Any from typing import TYPE_CHECKING @@ -14,7 +15,9 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names -from ._preferential_history import serialize_preference_history +from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY +from ._preferential_history import ChooseWorstHistory +from ._preferential_history import History from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -161,6 +164,29 @@ def serialize_study_detail( return serialized +def serialize_preference_history( + system_attrs: dict[str, Any], +) -> list[History]: + histories: list[History] = [] + for k, v in system_attrs.items(): + if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): + continue + choice: dict[str, Any] = json.loads(v) + if choice["mode"] == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": choice["id"], + "preference_id": choice["preference_id"], + "timestamp": choice["timestamp"], + "candidates": choice["candidates"], + "clicked": choice["clicked"], + } + histories.append(history) + + histories.sort(key=lambda c: datetime.fromisoformat(c["timestamp"])) + return histories + + def serialize_frozen_trial( study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any] ) -> dict[str, Any]: diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 00347d7e..e65a935d 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -16,8 +16,8 @@ def report_preferences( storage: BaseStorage, preferences: list[tuple[int, int]], ) -> str: - preference_uuid = str(uuid.uuid4()) - key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_uuid + preference_id = str(uuid.uuid4()) + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_id storage.set_study_system_attr( study_id=study_id, key=key, @@ -31,7 +31,7 @@ def report_preferences( trial_id = trials[number]._trial_id if trials[number].state != TrialState.COMPLETE: storage.set_trial_state_values(trial_id, TrialState.COMPLETE, values) - return preference_uuid + return preference_id def get_preferences( diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index f21fd326..3bc5a750 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -208,7 +208,7 @@ export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({ > {preference_histories.reverse().map((choice) => ( diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index b23dee9e..7f184e51 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -2,6 +2,7 @@ from __future__ import annotations from typing import Callable +from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import report_history from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study @@ -25,20 +26,20 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) report_history( study_id=study_id, storage=storage, - input_data={ - "mode": "ChooseWorst", - "candidates": [0, 1, 2], - "clicked": 1, - }, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 1, 2], + clicked=1, + ), ) report_history( study_id=study_id, storage=storage, - input_data={ - "mode": "ChooseWorst", - "candidates": [0, 2, 3, 4], - "clicked": 0, - }, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 2, 3, 4], + clicked=0, + ), ) history = serialize_preference_history(storage.get_study_system_attrs(study_id)) sys_attrs = storage.get_study_system_attrs(study_id) From 5ed5ced51b0ab54024baa6080750ae4235b3bbbe Mon Sep 17 00:00:00 2001 From: Contramundum Date: Wed, 6 Sep 2023 10:51:04 +0900 Subject: [PATCH 18/30] Add enqueue_trial --- optuna_dashboard/preferential/_study.py | 51 +++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index ce691f42..e61f5a3c 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -182,6 +182,57 @@ class PreferentialStudy: """ self._study.add_trials(trials) + def enqueue_trial( + self, + params: dict[str, Any], + user_attrs: dict[str, Any] | None = None, + skip_if_exists: bool = False, + ) -> None: + """Enqueue a trial with given parameter values. + + You can fix the next sampling parameters which will be evaluated in your + objective function. + + Example: + + .. testcode:: + + import optuna + + + def objective(trial): + x = trial.suggest_float("x", 0, 10) + return x**2 + + + study = optuna.create_study() + study.enqueue_trial({"x": 5}) + study.enqueue_trial({"x": 0}, user_attrs={"memo": "optimal"}) + study.optimize(objective, n_trials=2) + + assert study.trials[0].params == {"x": 5} + assert study.trials[1].params == {"x": 0} + assert study.trials[1].user_attrs == {"memo": "optimal"} + + Args: + params: + Parameter values to pass your objective function. + user_attrs: + A dictionary of user-specific attributes other than ``params``. + skip_if_exists: + When :obj:`True`, prevents duplicate trials from being enqueued again. + + .. note:: + This method might produce duplicated trials if called simultaneously + by multiple processes at the same time with same ``params`` dict. + + .. seealso:: + + Please refer to :ref:`enqueue_trial_tutorial` for the tutorial of specifying + hyperparameters manually. + """ + self._study.enqueue_trial(params, user_attrs, skip_if_exists) + def report_preference( self, better_trials: FrozenTrial | list[FrozenTrial], From 11cf11e589f33b9f1ecb4c70919de0192099d0fd Mon Sep 17 00:00:00 2001 From: contramundum53 Date: Wed, 6 Sep 2023 11:23:53 +0900 Subject: [PATCH 19/30] Update optuna_dashboard/preferential/_study.py Co-authored-by: c-bata --- optuna_dashboard/preferential/_study.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index e61f5a3c..eeed0a5b 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -225,11 +225,6 @@ class PreferentialStudy: .. note:: This method might produce duplicated trials if called simultaneously by multiple processes at the same time with same ``params`` dict. - - .. seealso:: - - Please refer to :ref:`enqueue_trial_tutorial` for the tutorial of specifying - hyperparameters manually. """ self._study.enqueue_trial(params, user_attrs, skip_if_exists) From 965673c8390c7e694641bb432002c12f88565db5 Mon Sep 17 00:00:00 2001 From: contramundum53 Date: Wed, 6 Sep 2023 11:24:00 +0900 Subject: [PATCH 20/30] Update optuna_dashboard/preferential/_study.py Co-authored-by: c-bata --- optuna_dashboard/preferential/_study.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index eeed0a5b..caa1896d 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -193,26 +193,12 @@ class PreferentialStudy: You can fix the next sampling parameters which will be evaluated in your objective function. - Example: + .. seealso:: - .. testcode:: + See `Study.enqueue_trials`_ for details. - import optuna - - - def objective(trial): - x = trial.suggest_float("x", 0, 10) - return x**2 - - - study = optuna.create_study() - study.enqueue_trial({"x": 5}) - study.enqueue_trial({"x": 0}, user_attrs={"memo": "optimal"}) - study.optimize(objective, n_trials=2) - - assert study.trials[0].params == {"x": 5} - assert study.trials[1].params == {"x": 0} - assert study.trials[1].user_attrs == {"memo": "optimal"} + .. _Study.get_trials: https://optuna.readthedocs.io/en/stable/reference/\ + generated/optuna.study.Study.html#optuna.study.Study.enqueue_trials Args: params: From a48f04d6ed520498c1a954b4444b485b4b7b33cd Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 6 Sep 2023 11:30:59 +0900 Subject: [PATCH 21/30] fix for pytest --- python_tests/test_api.py | 2 +- python_tests/test_preferential_history.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 0811c229..e50f2501 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -159,7 +159,7 @@ class APITestCase(TestCase): def test_report_preference_when_typo_mode(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(storage=storage, n_generate=3) for _ in range(3): trial = study.ask() study.mark_comparison_ready(trial) diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 7f184e51..ab524b90 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -15,7 +15,7 @@ from .storage_supplier import StorageSupplier @parametrize_storages def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(storage=storage, n_generate=5) for _ in range(5): trial = study.ask() trial.suggest_float("x", 0, 1) From 4f9c7df3418ad3d874019ba9a0ccaf4d3e2f6772 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 6 Sep 2023 14:11:35 +0900 Subject: [PATCH 22/30] fix by review --- optuna_dashboard/_preferential_history.py | 4 ---- optuna_dashboard/_serializer.py | 5 +++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index a246c166..6b81b9bb 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass from datetime import datetime import json -from typing import Any from typing import TYPE_CHECKING import uuid @@ -31,9 +30,6 @@ if TYPE_CHECKING: }, ) History = ChooseWorstHistory -else: - ChooseWorstHistory = Any - History = Any @dataclass diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index a0999b1f..06b53c42 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -16,8 +16,6 @@ from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY -from ._preferential_history import ChooseWorstHistory -from ._preferential_history import History from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -26,6 +24,9 @@ if TYPE_CHECKING: from typing import Literal from typing import TypedDict + from ._preferential_history import ChooseWorstHistory + from ._preferential_history import History + Attribute = TypedDict( "Attribute", { From e39357bd20a89d804507019a091b3fdc077cf057 Mon Sep 17 00:00:00 2001 From: c-bata Date: Wed, 6 Sep 2023 18:01:32 +0900 Subject: [PATCH 23/30] Support user-defined plotly figures --- optuna_dashboard/__init__.py | 1 + optuna_dashboard/_app.py | 4 + optuna_dashboard/_custom_plot_data.py | 115 ++++++++++++++++++ optuna_dashboard/_serializer.py | 5 + optuna_dashboard/ts/apiClient.ts | 2 + .../ts/components/StudyHistory.tsx | 12 ++ .../ts/components/UserDefinedPlot.tsx | 16 +++ optuna_dashboard/ts/types/index.d.ts | 6 + pyproject.toml | 1 + python_tests/test_custom_plot_data.py | 63 ++++++++++ python_tests/test_serializers.py | 4 +- 11 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 optuna_dashboard/_custom_plot_data.py create mode 100644 optuna_dashboard/ts/components/UserDefinedPlot.tsx create mode 100644 python_tests/test_custom_plot_data.py diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 3d363cf4..493af736 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -1,5 +1,6 @@ from ._app import run_server # noqa from ._app import wsgi # noqa +from ._custom_plot_data import save_plotly_graph_object # noqa from ._form_widget import ChoiceWidget # noqa from ._form_widget import dict_to_form_widget # noqa from ._form_widget import ObjectiveChoiceWidget # noqa diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 5f433072..c32c2061 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -25,6 +25,7 @@ from . import _note as note from ._bottle_util import BottleViewReturn from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property +from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials from ._preferential_history import NewHistory @@ -214,6 +215,8 @@ def create_app( union_user_attrs, has_intermediate_values, ) = get_cached_extra_study_property(study_id, trials) + + plotly_graph_objects = get_plotly_graph_objects(system_attrs) return serialize_study_detail( summary, best_trials, @@ -222,6 +225,7 @@ def create_app( union, union_user_attrs, has_intermediate_values, + plotly_graph_objects, ) @app.get("/api/studies//param_importances") diff --git a/optuna_dashboard/_custom_plot_data.py b/optuna_dashboard/_custom_plot_data.py new file mode 100644 index 00000000..d669d4ad --- /dev/null +++ b/optuna_dashboard/_custom_plot_data.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import math +from typing import TYPE_CHECKING +import uuid + +from optuna import Study + + +if TYPE_CHECKING: + from typing import Any + + from optuna.storages import BaseStorage + import plotly.graph_objs as go + + +SYSTEM_ATTR_PLOT_DATA = "dashboard:plot_data:" +SYSTEM_ATTR_MAX_LENGTH = 2045 + + +def save_plotly_graph_object( + study: Study, figure: go.Figure, *, graph_object_id: str | None = None +) -> str: + """Save the user-defined plotly's graph object to the study. + + Example: + + .. code-block:: python + + import optuna + from optuna_dashboard import save_plotly_graph_object + + def objective(trial): + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + study.optimize(objective, n_trials=100) + + figure = optuna.visualization.plot_optimization_history(study) + save_plotly_graph_object(study, figure) + + Args: + study: + Target study object. + plot_data: + The plotly's graph object to save. + graph_object_id: + Unique identifier of the graph object. If specified, the graph object is overwritten. + + Returns: + The graph object ID. + """ + storage = study._storage + study_id = study._study_id + + graph_object_id = graph_object_id or str(uuid.uuid4()) + key = SYSTEM_ATTR_PLOT_DATA + graph_object_id + ":" + plot_data_json_str = figure.to_json() + save_graph_object_json(storage, study_id, key, plot_data_json_str) + return graph_object_id + + +def save_graph_object_json( + storage: BaseStorage, study_id: int, key_prefix: str, plot_data_json_str: str +) -> None: + plot_data_system_attrs = split_plot_data(plot_data_json_str, key_prefix) + for k, v in plot_data_system_attrs.items(): + storage.set_study_system_attr(study_id, k, v) + + # Clear previous graph object attributes + study_system_attrs = storage.get_study_system_attrs(study_id) + all_plot_data_system_attrs = [k for k in study_system_attrs if k.startswith(key_prefix)] + if len(all_plot_data_system_attrs) > len(plot_data_system_attrs): + for i in range(len(plot_data_system_attrs), len(all_plot_data_system_attrs)): + storage.set_study_system_attr(study_id, f"{key_prefix}{i}", "") + + +def list_graph_object_ids(system_attrs: dict[str, Any]) -> list[str]: + titles = set() + for key in system_attrs: + if not key.startswith(SYSTEM_ATTR_PLOT_DATA): + continue + + s = key.split(":", maxsplit=2) # e.g. ["dashboard", "plot_data", "Optimization History:1"] + if len(s) != 3: + continue + # Please note that title may contain ":". + title = s[2].rsplit(":", maxsplit=1)[0] + titles.add(title) + return list(titles) + + +def get_plotly_graph_objects(system_attrs: dict[str, Any]) -> dict[str, str]: + graph_objects = {} + for title in list_graph_object_ids(system_attrs): + key_prefix = SYSTEM_ATTR_PLOT_DATA + title + ":" + plot_data_attrs = {k: v for k, v in system_attrs.items() if k.startswith(key_prefix)} + graph_objects[title] = concat_plot_data(plot_data_attrs, key_prefix) + return graph_objects + + +def split_plot_data(plot_data_str: str, key_prefix: str) -> dict[str, str]: + plot_data_len = len(plot_data_str) + attrs = {} + for i in range(math.ceil(plot_data_len / SYSTEM_ATTR_MAX_LENGTH)): + start = i * SYSTEM_ATTR_MAX_LENGTH + end = min((i + 1) * SYSTEM_ATTR_MAX_LENGTH, plot_data_len) + attrs[f"{key_prefix}{i}"] = plot_data_str[start:end] + return attrs + + +def concat_plot_data(plot_data_attrs: dict[str, str], key_prefix: str) -> str: + return "".join(plot_data_attrs[f"{key_prefix}{i}"] for i in range(len(plot_data_attrs))) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 06b53c42..19acbbd5 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -132,6 +132,7 @@ def serialize_study_detail( union: list[tuple[str, BaseDistribution]], union_user_attrs: list[tuple[str, bool]], has_intermediate_values: bool, + plotly_graph_objects: dict[str, str], ) -> dict[str, Any]: serialized: dict[str, Any] = { "name": summary.study_name, @@ -162,6 +163,10 @@ def serialize_study_detail( serialized["form_widgets"] = form_widgets if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) + serialized["plotly_graph_objects"] = [ + {"id": id_, "graph_object": graph_object} + for id_, graph_object in plotly_graph_objects.items() + ] return serialized diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index e23fc2ff..e62e0e42 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -93,6 +93,7 @@ interface StudyDetailResponse { objective_names?: string[] form_widgets?: FormWidgets preference_history?: PreferenceHistoryResponce[] + plotly_graph_objects: PlotlyGraphObject[] } export const getStudyDetailAPI = ( @@ -131,6 +132,7 @@ export const getStudyDetailAPI = ( preference_history: res.data.preference_history?.map( convertPreferenceHistory ), + plotly_graph_objects: res.data.plotly_graph_objects, } }) } diff --git a/optuna_dashboard/ts/components/StudyHistory.tsx b/optuna_dashboard/ts/components/StudyHistory.tsx index b47c557a..5cca3671 100644 --- a/optuna_dashboard/ts/components/StudyHistory.tsx +++ b/optuna_dashboard/ts/components/StudyHistory.tsx @@ -15,6 +15,7 @@ import { GraphIntermediateValues } from "./GraphIntermediateValues" import Grid2 from "@mui/material/Unstable_Grid2" import { DataGrid, DataGridColumn } from "./DataGrid" import { GraphHyperparameterImportance } from "./GraphHyperparameterImportances" +import { UserDefinedPlot } from "./UserDefinedPlot" import { BestTrialsCard } from "./BestTrialsCard" import { useStudyDetailValue, @@ -102,6 +103,17 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => { /> + {studyDetail !== null && + studyDetail.plotly_graph_objects.map((go) => ( + + + + ))} {studyDetail !== null && studyDetail.directions.length == 1 && diff --git a/optuna_dashboard/ts/components/UserDefinedPlot.tsx b/optuna_dashboard/ts/components/UserDefinedPlot.tsx new file mode 100644 index 00000000..2a6f98db --- /dev/null +++ b/optuna_dashboard/ts/components/UserDefinedPlot.tsx @@ -0,0 +1,16 @@ +import * as plotly from "plotly.js-dist-min" +import React, { FC, useEffect } from "react" +import { Box } from "@mui/material" + +export const UserDefinedPlot: FC<{ + graphObject: PlotlyGraphObject +}> = ({ graphObject }) => { + const plotDomId = `user-defined-plot:${graphObject.id}` + + useEffect(() => { + const parsed = JSON.parse(graphObject.graph_object) + plotly.react(plotDomId, parsed.data, parsed.layout) + }, [graphObject]) + + return +} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 646d64cf..b7b35797 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -182,6 +182,11 @@ type FormWidgets = widgets: UserAttrFormWidget[] } +type PlotlyGraphObject = { + id: string + graph_object: string +} + type StudyDetail = { id: number name: string @@ -199,6 +204,7 @@ type StudyDetail = { objective_names?: string[] form_widgets?: FormWidgets preference_history?: PreferenceHistory[] + plotly_graph_objects: PlotlyGraphObject[] } type StudyDetails = { diff --git a/pyproject.toml b/pyproject.toml index 0555f701..9b7ce731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ docs = [ test = [ "coverage", + "plotly", "pytest", "moto[s3]", ] diff --git a/python_tests/test_custom_plot_data.py b/python_tests/test_custom_plot_data.py new file mode 100644 index 00000000..fdf738d7 --- /dev/null +++ b/python_tests/test_custom_plot_data.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from unittest.mock import patch + +import optuna +from optuna_dashboard import _custom_plot_data as custom_plot_data +from optuna_dashboard import save_plotly_graph_object + + +def get_dummy_study() -> optuna.Study: + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + optuna.logging.set_verbosity(optuna.logging.ERROR) + study.optimize(objective, n_trials=100) + return study + + +def test_save_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 2 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +def test_update_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object( + dummy_study, plot_data, graph_object_id=graph_object_id + ) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index a90e0de7..72db7b26 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -29,7 +29,7 @@ def test_get_study_detail_is_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert study_detail["is_preferential"] @@ -40,7 +40,7 @@ def test_get_study_detail_is_not_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert not study_detail["is_preferential"] From 5b64b38bf21a9d5dd157d529d70fe17cdb900ed6 Mon Sep 17 00:00:00 2001 From: c-bata Date: Wed, 6 Sep 2023 18:15:24 +0900 Subject: [PATCH 24/30] Fix flake8 error --- python_tests/test_custom_plot_data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python_tests/test_custom_plot_data.py b/python_tests/test_custom_plot_data.py index fdf738d7..4d01af96 100644 --- a/python_tests/test_custom_plot_data.py +++ b/python_tests/test_custom_plot_data.py @@ -1,7 +1,5 @@ from __future__ import annotations -from unittest.mock import patch - import optuna from optuna_dashboard import _custom_plot_data as custom_plot_data from optuna_dashboard import save_plotly_graph_object From 7c5e0a84e0cf3db9e5ea5ca3d7c53c18ee82d3c8 Mon Sep 17 00:00:00 2001 From: c-bata Date: Wed, 6 Sep 2023 18:38:29 +0900 Subject: [PATCH 25/30] Update docs --- docs/api.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api.rst b/docs/api.rst index 09a9c14b..aadd2718 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -14,6 +14,7 @@ General APIs optuna_dashboard.wsgi optuna_dashboard.set_objective_names optuna_dashboard.save_note + optuna_dashboard.save_plotly_graph_object Human-in-the-loop ----------------- From 5f1d3bb2c416e765dd45bc77afc43893950dff0e Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 7 Sep 2023 10:31:10 +0900 Subject: [PATCH 26/30] Validate graph object id --- optuna_dashboard/_custom_plot_data.py | 21 +++++++++++++++++++++ python_tests/test_custom_plot_data.py | 25 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/optuna_dashboard/_custom_plot_data.py b/optuna_dashboard/_custom_plot_data.py index d669d4ad..a88fc0a4 100644 --- a/optuna_dashboard/_custom_plot_data.py +++ b/optuna_dashboard/_custom_plot_data.py @@ -48,10 +48,14 @@ def save_plotly_graph_object( The plotly's graph object to save. graph_object_id: Unique identifier of the graph object. If specified, the graph object is overwritten. + This must be a valid HTML id attribute value. Returns: The graph object ID. """ + if graph_object_id is not None and not is_valid_html_name(graph_object_id): + raise ValueError("graph_object_id must be a valid HTML id attribute value.") + storage = study._storage study_id = study._study_id @@ -113,3 +117,20 @@ def split_plot_data(plot_data_str: str, key_prefix: str) -> dict[str, str]: def concat_plot_data(plot_data_attrs: dict[str, str], key_prefix: str) -> str: return "".join(plot_data_attrs[f"{key_prefix}{i}"] for i in range(len(plot_data_attrs))) + + +def is_valid_html_name(graph_object_id: str) -> bool: + if len(graph_object_id) == 0: + return False + + # Must begin with a letter [A-Za-z] + if not ("a" <= graph_object_id[0] <= "z" or "A" <= graph_object_id[0] <= "Z"): + return False + + # Can only contain letters [A-Za-z], numbers [0-9], hyphens ("-"), underscores ("_"), colons, and periods. + if not all( + "a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9" or c in ("-", "_", ":", ".") + for c in graph_object_id[1:] + ): + return False + return True diff --git a/python_tests/test_custom_plot_data.py b/python_tests/test_custom_plot_data.py index 4d01af96..f73f7097 100644 --- a/python_tests/test_custom_plot_data.py +++ b/python_tests/test_custom_plot_data.py @@ -3,6 +3,7 @@ from __future__ import annotations import optuna from optuna_dashboard import _custom_plot_data as custom_plot_data from optuna_dashboard import save_plotly_graph_object +import pytest def get_dummy_study() -> optuna.Study: @@ -59,3 +60,27 @@ def test_update_plotly_graph_object() -> None: plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) assert len(plot_data_dict) == 1 assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +@pytest.mark.parametrize( + "name", + [ + "a", + "a1-:_.", + ], +) +def test_is_valid_html_name(name): + assert custom_plot_data.is_valid_html_name(name) + + +@pytest.mark.parametrize( + "name", + [ + "0", + "a,", + "a b", + "aあいうえお", + ], +) +def test_is_invalid_html_name(name): + assert not custom_plot_data.is_valid_html_name(name) From 0caa0e6ac4c2c57325a7fe7d7b221751670e3255 Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 7 Sep 2023 10:40:02 +0900 Subject: [PATCH 27/30] Make UserDefinedPlot half widths --- .../ts/components/StudyHistory.tsx | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/optuna_dashboard/ts/components/StudyHistory.tsx b/optuna_dashboard/ts/components/StudyHistory.tsx index 5cca3671..907acd1a 100644 --- a/optuna_dashboard/ts/components/StudyHistory.tsx +++ b/optuna_dashboard/ts/components/StudyHistory.tsx @@ -103,17 +103,6 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => { /> - {studyDetail !== null && - studyDetail.plotly_graph_objects.map((go) => ( - - - - ))} {studyDetail !== null && studyDetail.directions.length == 1 && @@ -136,6 +125,16 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => { + {studyDetail !== null && + studyDetail.plotly_graph_objects.map((go) => ( + + + + + + + + ))} From e3b7e1d89360357b7a353dd20ad3b62338305711 Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 7 Sep 2023 10:48:11 +0900 Subject: [PATCH 28/30] Avoid to crash the whole page when given invalid figures --- optuna_dashboard/ts/components/UserDefinedPlot.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/ts/components/UserDefinedPlot.tsx b/optuna_dashboard/ts/components/UserDefinedPlot.tsx index 2a6f98db..029c4b57 100644 --- a/optuna_dashboard/ts/components/UserDefinedPlot.tsx +++ b/optuna_dashboard/ts/components/UserDefinedPlot.tsx @@ -8,8 +8,13 @@ export const UserDefinedPlot: FC<{ const plotDomId = `user-defined-plot:${graphObject.id}` useEffect(() => { - const parsed = JSON.parse(graphObject.graph_object) - plotly.react(plotDomId, parsed.data, parsed.layout) + try { + const parsed = JSON.parse(graphObject.graph_object) + plotly.react(plotDomId, parsed.data, parsed.layout) + } catch (e) { + // Avoid to crash the whole page when given invalid grpah objects. + console.error(e) + } }, [graphObject]) return From 167a4de6c4f16a7d61cdf67c371feef74d8acbce Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 7 Sep 2023 10:53:08 +0900 Subject: [PATCH 29/30] Fix tests --- optuna_dashboard/_custom_plot_data.py | 12 +++++------- python_tests/test_custom_plot_data.py | 10 +++++----- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/optuna_dashboard/_custom_plot_data.py b/optuna_dashboard/_custom_plot_data.py index a88fc0a4..a4dfa4af 100644 --- a/optuna_dashboard/_custom_plot_data.py +++ b/optuna_dashboard/_custom_plot_data.py @@ -53,7 +53,7 @@ def save_plotly_graph_object( Returns: The graph object ID. """ - if graph_object_id is not None and not is_valid_html_name(graph_object_id): + if graph_object_id is not None and not is_valid_graph_object_id(graph_object_id): raise ValueError("graph_object_id must be a valid HTML id attribute value.") storage = study._storage @@ -119,18 +119,16 @@ def concat_plot_data(plot_data_attrs: dict[str, str], key_prefix: str) -> str: return "".join(plot_data_attrs[f"{key_prefix}{i}"] for i in range(len(plot_data_attrs))) -def is_valid_html_name(graph_object_id: str) -> bool: +def is_valid_graph_object_id(graph_object_id: str) -> bool: if len(graph_object_id) == 0: return False - # Must begin with a letter [A-Za-z] - if not ("a" <= graph_object_id[0] <= "z" or "A" <= graph_object_id[0] <= "Z"): - return False - - # Can only contain letters [A-Za-z], numbers [0-9], hyphens ("-"), underscores ("_"), colons, and periods. + # Can only contain letters [A-Za-z], numbers [0-9], hyphens ("-"), underscores ("_"), + # colons, and periods. if not all( "a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9" or c in ("-", "_", ":", ".") for c in graph_object_id[1:] ): return False + # Unlike HTML id attribute, graph object id can begin with a letter [A-Za-z] return True diff --git a/python_tests/test_custom_plot_data.py b/python_tests/test_custom_plot_data.py index f73f7097..3dcfc856 100644 --- a/python_tests/test_custom_plot_data.py +++ b/python_tests/test_custom_plot_data.py @@ -65,22 +65,22 @@ def test_update_plotly_graph_object() -> None: @pytest.mark.parametrize( "name", [ + "0", "a", "a1-:_.", ], ) -def test_is_valid_html_name(name): - assert custom_plot_data.is_valid_html_name(name) +def test_is_valid_graph_object_id(name: str) -> None: + assert custom_plot_data.is_valid_graph_object_id(name) @pytest.mark.parametrize( "name", [ - "0", "a,", "a b", "aあいうえお", ], ) -def test_is_invalid_html_name(name): - assert not custom_plot_data.is_valid_html_name(name) +def test_is_invalid_graph_object_id(name: str) -> None: + assert not custom_plot_data.is_valid_graph_object_id(name) From 7b0dce75aabe0ad254b14a1bf7bdf9a01fc72932 Mon Sep 17 00:00:00 2001 From: keisuke umezawa Date: Thu, 7 Sep 2023 13:52:57 +0900 Subject: [PATCH 30/30] Update python-coverage.yml --- .github/workflows/python-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-coverage.yml b/.github/workflows/python-coverage.yml index 4bda6350..c96f506b 100644 --- a/.github/workflows/python-coverage.yml +++ b/.github/workflows/python-coverage.yml @@ -45,4 +45,4 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml - fail_ci_if_error: true + fail_ci_if_error: false