From b09e223e17710db2292ba9845bc291b0a971f1e0 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 1 Sep 2023 12:03:45 +0900 Subject: [PATCH] modify API --- optuna_dashboard/_app.py | 24 ++--- optuna_dashboard/_preferential_history.py | 92 +++++++++---------- optuna_dashboard/ts/action.ts | 6 +- optuna_dashboard/ts/apiClient.ts | 14 +-- .../ts/components/PreferenceHistory.tsx | 10 +- .../ts/components/PreferentialTrials.tsx | 11 +-- optuna_dashboard/ts/types/index.d.ts | 4 +- python_tests/test_api.py | 5 +- python_tests/test_preferential_history.py | 62 +++++++------ 9 files changed, 109 insertions(+), 119 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 87fbaa9a..60eccd8d 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,7 +28,6 @@ from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preferential_history import ChooseWorstHistory from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail @@ -281,22 +280,19 @@ def create_app( if clicked == -1: response.status = 400 return {"reason": "`clicked` should be specified."} - - if mode == "ChooseWorst": - history = ChooseWorstHistory.create( - study_id, - storage, - { - "mode": mode, - "candidates": candidates, - "clicked": clicked, - }, - ) - else: + if mode != "ChooseWorst": response.status = 400 return {"reason": "`mode` should be 'ChooseWorst'."} - report_history(study_id, storage, history) + report_history( + study_id, + storage, + { + "mode": mode, + "candidates": candidates, + "clicked": clicked, + }, + ) response.status = 204 return {} diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 52f10c3b..5c930932 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -10,7 +10,6 @@ import uuid from optuna.storages import BaseStorage -from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE from .preferential._system_attrs import report_preferences @@ -19,8 +18,8 @@ _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" if TYPE_CHECKING: from typing import TypedDict - NewChooseWorstHistoryJSON = TypedDict( - "NewChooseWorstHistoryJSON", + NewHistoryJSON = TypedDict( + "NewHistoryJSON", { "mode": Literal["ChooseWorst"], "candidates": list[int], @@ -28,8 +27,6 @@ if TYPE_CHECKING: }, ) - NewHistoryJSON = NewChooseWorstHistoryJSON - @dataclass(frozen=True) class ChooseWorstHistory: @@ -50,45 +47,6 @@ class ChooseWorstHistory: "clicked": self.clicked, } - def to_response(self, system_attrs: dict[str, Any]) -> dict[str, Any]: - return { - "uuid": self.uuid, - "preferences": system_attrs.get( - _SYSTEM_ATTR_PREFIX_PREFERENCE + self.preference_uuid, [] - ), - "timestamp": self.timestamp.isoformat(), - "candidates": self.candidates, - "clicked": self.clicked, - } - - @classmethod - def from_dict(cls, d: dict[str, Any]) -> ChooseWorstHistory: - return ChooseWorstHistory( - mode="ChooseWorst", - uuid=d["uuid"], - preference_uuid=d["preference_uuid"], - timestamp=datetime.fromisoformat(d["timestamp"]), - candidates=d["candidates"], - clicked=d["clicked"], - ) - - @classmethod - def create(cls, study_id: int, storage: BaseStorage, d: NewHistoryJSON) -> ChooseWorstHistory: - preferences = [(best, d["clicked"]) for best in d["candidates"] if best != d["clicked"]] - preference_uuid = report_preferences( - study_id=study_id, - storage=storage, - preferences=preferences, - ) - return ChooseWorstHistory( - mode="ChooseWorst", - uuid=str(uuid.uuid4()), - preference_uuid=preference_uuid, - timestamp=datetime.now(), - candidates=d["candidates"], - clicked=d["clicked"], - ) - History = ChooseWorstHistory @@ -96,9 +54,36 @@ History = ChooseWorstHistory def report_history( study_id: int, storage: BaseStorage, - history: History, + input_data: NewHistoryJSON, ) -> None: - key = _SYSTEM_ATTR_PREFIX_HISTORY + history.uuid + preferences = [] + if input_data["mode"] == "ChooseWorst": + preferences = [ + (best, input_data["clicked"]) + for best in input_data["candidates"] + if best != input_data["clicked"] + ] + else: + assert False, f"Unknown mode: {input_data['mode']}" + + preference_uuid = report_preferences( + study_id=study_id, + storage=storage, + preferences=preferences, + ) + history_uuid = str(uuid.uuid4()) + + if input_data["mode"] == "ChooseWorst": + history = ChooseWorstHistory( + mode="ChooseWorst", + uuid=history_uuid, + preference_uuid=preference_uuid, + timestamp=datetime.now(), + candidates=input_data["candidates"], + clicked=input_data["clicked"], + ) + + key = _SYSTEM_ATTR_PREFIX_HISTORY + history_uuid storage.set_study_system_attr( study_id=study_id, key=key, @@ -115,9 +100,18 @@ def serialize_preference_history( continue choice: dict[str, Any] = json.loads(v) if choice["mode"] == "ChooseWorst": - histories.append(ChooseWorstHistory.from_dict(choice)) + histories.append( + ChooseWorstHistory( + mode="ChooseWorst", + uuid=choice["uuid"], + preference_uuid=choice["preference_uuid"], + timestamp=datetime.fromisoformat(choice["timestamp"]), + candidates=choice["candidates"], + clicked=choice["clicked"], + ) + ) else: - assert False + assert False, f"Unknown mode: {choice['mode']}" histories.sort(key=lambda c: c.timestamp) - return [history.to_response(system_attrs) for history in histories] + return [history.to_dict() for history in histories] diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index 017751fa..e1896178 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -587,10 +587,10 @@ export const actionCreator = () => { const updatePreference = ( study_id: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ) => { - reportPreferenceAPI(study_id, best_trials, worst_trials).catch((err) => { + reportPreferenceAPI(study_id, candidates, clicked).catch((err) => { const reason = err.response?.data.reason enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { variant: "error", diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index d3cddc89..9c72f739 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -58,7 +58,7 @@ const convertTrialResponse = (res: TrialResponse): Trial => { interface PreferenceChoiceResponce { uuid: string candidates: number[] - preferences: number[][] + clicked: number mode: PreferenceFeedbackMode timestamp: string } @@ -68,8 +68,8 @@ const convertPreferenceChoice = ( ): PreferenceChoice => { return { uuid: res.uuid, - candidate_trials: res.candidates, - preferences: res.preferences, + candidates: res.candidates, + clicked: res.clicked, feedback_mode: res.mode, timestamp: new Date(res.timestamp), } @@ -338,13 +338,13 @@ export const getParamImportances = ( export const reportPreferenceAPI = ( studyId: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ): Promise => { return axiosInstance .post(`/api/studies/${studyId}/preference`, { - candidates: best_trials.concat(worst_trials), - clicked: worst_trials[0], + candidates: candidates, + clicked: clicked, mode: "ChooseWorst", }) .then(() => { diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 1e4c9346..e95a4cb1 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -138,13 +138,7 @@ const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ trials, }) => { const theme = useTheme() - const worst_trials = new Set(choice.preferences.map((pair) => pair[1])) - console.log( - worst_trials, - choice.candidate_trials, - choice.preferences, - choice.preferences.map((pair) => pair[1]) - ) + const worst_trials = new Set([choice.clicked]) return ( @@ -163,7 +157,7 @@ const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ flexDirection: "row", }} > - {choice.candidate_trials.map((trial_num, index) => ( + {choice.candidates.map((trial_num, index) => ( void -}> = ({ trial, studyDetail, hideTrial }) => { +}> = ({ trial, candidates, hideTrial }) => { const theme = useTheme() const action = actionCreator() const trialWidth = 500 @@ -80,10 +80,7 @@ const PreferentialTrial: FC<{ aria-label="trial-button" onClick={() => { hideTrial() - const best_trials = studyDetail.best_trials - .map((t) => t.number) - .filter((t) => t !== trial.number) - action.updatePreference(trial.study_id, best_trials, [trial.number]) + action.updatePreference(trial.study_id, candidates, trial.number) }} sx={{ padding: 0, @@ -243,7 +240,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ trial.number === t)} - studyDetail={studyDetail} + candidates={displayTrials.numbers.filter((n) => n !== -1)} hideTrial={() => { hideTrial(t) }} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 358cb17c..da90ad1d 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -211,8 +211,8 @@ type StudyParamImportance = { type PreferenceChoice = { uuid: string - candidate_trials: number[] - preferences: number[][] + candidates: number[] + clicked: number feedback_mode: PreferenceFeedbackMode timestamp: Date } diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 8afa1021..7d732e37 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -138,8 +138,9 @@ class APITestCase(TestCase): "POST", body=json.dumps( { - "candidate_trials": [0, 1, 2], - "preferentials": [[0, 1], [2, 1]], + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 1, } ), content_type="application/json", diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 3e1c73f4..3448abc3 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -3,7 +3,8 @@ from __future__ import annotations from datetime import datetime from typing import Callable -from optuna_dashboard._preferential_history import report_choice +from optuna_dashboard._preferential_history import report_history +from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study @@ -21,34 +22,41 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) study.mark_comparison_ready(trial) study_id = study._study._study_id - report_choice( - study_id=study_id, - storage=storage, - candidate_trials=[0, 2, 3, 4], - preferences=[(2, 0), (3, 0), (4, 0)], - timestamp=datetime(2020, 1, 1, 10, 0, 1), - ) - report_choice( - study_id=study_id, - storage=storage, - candidate_trials=[0, 1, 2], - preferences=[(0, 1), (2, 1)], - timestamp=datetime(2020, 1, 1, 10, 0, 0), - ) + report_history( + study_id=study_id, + storage=storage, + input_data={ + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + }, + ) + report_history( + study_id=study_id, + storage=storage, + input_data={ + "mode": "ChooseWorst", + "candidates": [0, 2, 3, 4], + "clicked": 0, + }, + ) history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + sys_attrs = storage.get_study_system_attrs(study_id) assert len(history) == 2 - assert history[0]["candidate_trials"] == [0, 1, 2] - assert len(history[0]["preferences"]) == 2 + assert history[0]["candidates"] == [0, 1, 2] + assert history[0]["clicked"] == 1 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_uuid"]] + assert len(preferences) == 2 for i, (best, worst) in enumerate([(0, 1), (2, 1)]): - assert len(history[0]["preferences"][i]) == 2 - assert history[0]["preferences"][i][0] == best - assert history[0]["preferences"][i][1] == worst - assert history[0]["timestamp"] == "2020-01-01T10:00:00" - assert history[1]["candidate_trials"] == [0, 2, 3, 4] - assert len(history[1]["preferences"]) == 3 + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst + assert history[1]["candidates"] == [0, 2, 3, 4] + assert history[1]["clicked"] == 0 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_uuid"]] + assert len(preferences) == 3 for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): - assert len(history[1]["preferences"][i]) == 2 - assert history[1]["preferences"][i][0] == best - assert history[1]["preferences"][i][1] == worst - assert history[1]["timestamp"] == "2020-01-01T10:00:01" + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst