From adbbea39533b67f952071c4ae5eea7d68cb73dd4 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 16:23:06 +0900 Subject: [PATCH 1/4] wip: change hiding system on feedback screen in order to undo history --- .../ts/components/PreferentialTrials.tsx | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 08efd9a7..dc6e6c95 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -173,8 +173,8 @@ const PreferentialTrial: FC<{ } type DisplayTrials = { - numbers: number[] - last_number: number + display: number[] + clicked: number[] } export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ @@ -189,64 +189,70 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ const activeTrials = runningTrials.concat(studyDetail.best_trials) const [displayTrials, setDisplayTrials] = useState({ - numbers: activeTrials.map((t) => t.number), - last_number: Math.max(...activeTrials.map((t) => t.number), -1), + display: [], + clicked: [], }) const new_trails = activeTrials.filter( (t) => - displayTrials.last_number < t.number && - displayTrials.numbers.find((n) => n === t.number) === undefined + !displayTrials.display.includes(t.number) && + !displayTrials.clicked.includes(t.number) ) if (new_trails.length > 0) { - setDisplayTrials((display) => { - const numbers = [...display.numbers] + setDisplayTrials((prev) => { + const display = [...prev.display] + const clicked = [...prev.clicked] new_trails.map((t) => { - const index = numbers.findIndex((n) => n === -1) + const index = display.findIndex((n) => n === -1) if (index === -1) { - numbers.push(t.number) + display.push(t.number) + clicked.push(-1) } else { - numbers[index] = t.number + display[index] = t.number } }) return { - numbers: numbers, - last_number: Math.max(...numbers, -1), + display: display, + clicked: clicked, } }) } const hideTrial = (num: number) => { - setDisplayTrials((display) => { - const index = display.numbers.findIndex((n) => n === num) + setDisplayTrials((prev) => { + const index = prev.display.findIndex((n) => n === num) if (index === -1) { - return display + return prev } - const numbers = [...displayTrials.numbers] - numbers[index] = -1 + const display = [...prev.display] + const clicked = [...prev.clicked] + display[index] = -1 + clicked[index] = num return { - numbers: numbers, - last_number: display.last_number, + display: display, + clicked: clicked, } }) } return ( - - Which trial is the worst? - + + + Which trial is the worst? + + - {displayTrials.numbers.map((t, index) => ( + {displayTrials.display.map((t, index) => ( trial.number === t)} - candidates={displayTrials.numbers.filter((n) => n !== -1)} + candidates={displayTrials.display.filter((n) => n !== -1)} hideTrial={() => { hideTrial(t) }} From 2422555475586f71a129ae4f5f6a88ad8c65a5e8 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 11:53:07 +0900 Subject: [PATCH 2/4] modify fail trial --- .../ts/components/PreferentialTrials.tsx | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index dc6e6c95..ecb2a7e2 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -192,16 +192,22 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ display: [], clicked: [], }) - const new_trails = activeTrials.filter( + const newTrials = activeTrials.filter( (t) => !displayTrials.display.includes(t.number) && !displayTrials.clicked.includes(t.number) ) - if (new_trails.length > 0) { + const deleteTrials = displayTrials.display.filter( + (t) => t !== -1 && !activeTrials.map((t) => t.number).includes(t) + ) + console.log(deleteTrials) + if (newTrials.length > 0 || deleteTrials.length > 0) { setDisplayTrials((prev) => { - const display = [...prev.display] + const display = [...prev.display].map((t) => + deleteTrials.includes(t) ? -1 : t + ) const clicked = [...prev.clicked] - new_trails.map((t) => { + newTrials.map((t) => { const index = display.findIndex((n) => n === -1) if (index === -1) { display.push(t.number) @@ -250,7 +256,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {displayTrials.display.map((t, index) => ( trial.number === t)} candidates={displayTrials.display.filter((n) => n !== -1)} hideTrial={() => { From 3fdf22704c17dcb585629e0e6637104a8eda367e Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 14:33:25 +0900 Subject: [PATCH 3/4] fix feedback screen --- optuna_dashboard/_app.py | 6 ++++ optuna_dashboard/_serializer.py | 3 ++ optuna_dashboard/ts/apiClient.ts | 32 ++++++++++++------- .../ts/components/PreferentialTrials.tsx | 20 +++++++++--- optuna_dashboard/ts/types/index.d.ts | 3 ++ python_tests/test_serializers.py | 8 +++-- 6 files changed, 54 insertions(+), 18 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 0249de48..731de524 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -46,6 +46,7 @@ from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._study import get_best_trials as get_best_preferential_trials +from .preferential._system_attrs import get_skipped_trial_ids from .preferential._system_attrs import report_skip @@ -220,6 +221,10 @@ def create_app( ) = get_cached_extra_study_property(study_id, trials) plotly_graph_objects = get_plotly_graph_objects(system_attrs) + trials_id2number = {trial._trial_id: trial.number for trial in trials} + skipped_trials = [ + trials_id2number[trial_id] for trial_id in get_skipped_trial_ids(system_attrs) + ] return serialize_study_detail( summary, best_trials, @@ -229,6 +234,7 @@ def create_app( union_user_attrs, has_intermediate_values, plotly_graph_objects, + skipped_trials, ) @app.get("/api/studies//param_importances") diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 001f689a..9d2241be 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -19,6 +19,7 @@ from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._system_attrs import get_preferences +from .preferential._system_attrs import get_skipped_trial_ids from .preferential._system_attrs import is_preference_removed @@ -135,6 +136,7 @@ def serialize_study_detail( union_user_attrs: list[tuple[str, bool]], has_intermediate_values: bool, plotly_graph_objects: dict[str, str], + skipped_trials: list[int], ) -> dict[str, Any]: serialized: dict[str, Any] = { "name": summary.study_name, @@ -166,6 +168,7 @@ def serialize_study_detail( if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) + serialized["skipped_trials"] = skipped_trials serialized["plotly_graph_objects"] = [ {"id": id_, "graph_object": graph_object} for id_, graph_object in plotly_graph_objects.items() diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index c0ad90f8..fefabeb4 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -56,24 +56,30 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } interface PreferenceHistoryResponce { - id: string - preference_id: string - candidates: number[] - clicked: number - mode: PreferenceFeedbackMode - timestamp: string + history: { + id: string + preference_id: string + candidates: number[] + clicked: number + mode: PreferenceFeedbackMode + timestamp: string + preferences: [number, number][] + } + is_removed: boolean } const convertPreferenceHistory = ( res: PreferenceHistoryResponce ): PreferenceHistory => { return { - id: res.id, - preference_id: res.preference_id, - candidates: res.candidates, - clicked: res.clicked, - feedback_mode: res.mode, - timestamp: new Date(res.timestamp), + id: res.history.id, + preference_id: res.history.preference_id, + candidates: res.history.candidates, + clicked: res.history.clicked, + feedback_mode: res.history.mode, + timestamp: new Date(res.history.timestamp), + preferences: res.history.preferences, + is_removed: res.is_removed, } } @@ -95,6 +101,7 @@ interface StudyDetailResponse { preferences?: [number, number][] preference_history?: PreferenceHistoryResponce[] plotly_graph_objects: PlotlyGraphObject[] + skipped_trials?: number[] } export const getStudyDetailAPI = ( @@ -135,6 +142,7 @@ export const getStudyDetailAPI = ( convertPreferenceHistory ), plotly_graph_objects: res.data.plotly_graph_objects, + skipped_trials: res.data.skipped_trials ?? [], } }) } diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index ecb2a7e2..6b0ea1f6 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -7,6 +7,7 @@ import { CardContent, CardActions, CardActionArea, + CircularProgress, } from "@mui/material" import ClearIcon from "@mui/icons-material/Clear" import IconButton from "@mui/material/IconButton" @@ -111,7 +112,11 @@ const PreferentialTrial: FC<{ padding: theme.spacing(2), }} > - + {trial.note.body !== "" ? ( + + ) : ( + + )} = ({ } const theme = useTheme() - const runningTrials = studyDetail.trials.filter((t) => t.state === "Running") - const activeTrials = runningTrials.concat(studyDetail.best_trials) + const hiddenTrials = new Set( + studyDetail.preference_history + ?.map((p) => p.clicked) + .concat(studyDetail.skipped_trials) ?? [] + ) + const activeTrials = studyDetail.trials.filter( + (t) => + (t.state === "Running" || t.state === "Complete") && + !hiddenTrials.has(t.number) + ) const [displayTrials, setDisplayTrials] = useState({ display: [], @@ -200,7 +213,6 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ const deleteTrials = displayTrials.display.filter( (t) => t !== -1 && !activeTrials.map((t) => t.number).includes(t) ) - console.log(deleteTrials) if (newTrials.length > 0 || deleteTrials.length > 0) { setDisplayTrials((prev) => { const display = [...prev.display].map((t) => diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 81096248..32f7ebd1 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -206,6 +206,7 @@ type StudyDetail = { preferences?: [number, number][] preference_history?: PreferenceHistory[] plotly_graph_objects: PlotlyGraphObject[] + skipped_trials: number[] } type StudyDetails = { @@ -222,4 +223,6 @@ type PreferenceHistory = { clicked: number feedback_mode: PreferenceFeedbackMode timestamp: Date + preferences: [number, number][] + is_removed: boolean } diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index 72db7b26..ea1c3517 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -29,7 +29,9 @@ def test_get_study_detail_is_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) + study_detail = serialize_study_detail( + study_summary, [], study.trials, [], [], [], False, {}, [] + ) assert study_detail["is_preferential"] @@ -40,7 +42,9 @@ def test_get_study_detail_is_not_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) + study_detail = serialize_study_detail( + study_summary, [], study.trials, [], [], [], False, {}, [] + ) assert not study_detail["is_preferential"] From 1c3cb84d871e94b7441207aa350537545f8d0f53 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 14:40:52 +0900 Subject: [PATCH 4/4] fix by lint --- optuna_dashboard/_serializer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 9d2241be..d3cd786f 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -19,7 +19,6 @@ from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._system_attrs import get_preferences -from .preferential._system_attrs import get_skipped_trial_ids from .preferential._system_attrs import is_preference_removed