diff --git a/.github/workflows/python-coverage.yml b/.github/workflows/python-coverage.yml index 4bda6350..c96f506b 100644 --- a/.github/workflows/python-coverage.yml +++ b/.github/workflows/python-coverage.yml @@ -45,4 +45,4 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml - fail_ci_if_error: true + fail_ci_if_error: false diff --git a/docs/api.rst b/docs/api.rst index 09a9c14b..aadd2718 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -14,6 +14,7 @@ General APIs optuna_dashboard.wsgi optuna_dashboard.set_objective_names optuna_dashboard.save_note + optuna_dashboard.save_plotly_graph_object Human-in-the-loop ----------------- diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 3d363cf4..5bb3f301 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -1,5 +1,6 @@ from ._app import run_server # noqa from ._app import wsgi # noqa +from ._custom_plot_data import save_plotly_graph_object # noqa from ._form_widget import ChoiceWidget # noqa from ._form_widget import dict_to_form_widget # noqa from ._form_widget import ObjectiveChoiceWidget # noqa @@ -15,4 +16,4 @@ from ._note import get_note # noqa from ._note import save_note # noqa -__version__ = "0.12.0" +__version__ = "0.13.0b1" diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 1c439408..c32c2061 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -25,8 +25,11 @@ from . import _note as note from ._bottle_util import BottleViewReturn from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property +from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preferential_history import NewHistory +from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -40,7 +43,6 @@ from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._study import get_best_trials as get_best_preferential_trials -from .preferential._system_attrs import report_preferences from .preferential._system_attrs import report_skip @@ -213,6 +215,8 @@ def create_app( union_user_attrs, has_intermediate_values, ) = get_cached_extra_study_property(study_id, trials) + + plotly_graph_objects = get_plotly_graph_objects(system_attrs) return serialize_study_detail( summary, best_trials, @@ -221,6 +225,7 @@ def create_app( union, union_user_attrs, has_intermediate_values, + plotly_graph_objects, ) @app.get("/api/studies//param_importances") @@ -269,17 +274,34 @@ def create_app( @json_api_view def post_preference(study_id: int) -> dict[str, Any]: try: - best_trials = [int(d) for d in request.json.get("best_trials", [])] - worst_trials = [int(d) for d in request.json.get("worst_trials", [])] + mode = request.json.get("mode", "") + candidates = [int(d) for d in request.json.get("candidates", [])] + clicked = int(request.json.get("clicked", -1)) except ValueError: response.status = 400 - return {"reason": "best_trials and worst_trials must be an array of integers."} - if len(best_trials) == 0 or len(worst_trials) == 0: - response.status = 400 # Bad request - return {"reason": "You need to set best_trials and worst_trials"} + return { + "reason": ( + "`candidates` should be an array of integers and " + "`clicked` should be an integer." + ) + } - preferences = [(best, worst) for best in best_trials for worst in worst_trials] - report_preferences(study_id, storage, preferences) + if clicked == -1: + response.status = 400 + return {"reason": "`clicked` should be specified."} + if mode != "ChooseWorst": + response.status = 400 + return {"reason": "`mode` should be 'ChooseWorst'."} + + report_history( + study_id, + storage, + NewHistory( + mode=mode, + candidates=candidates, + clicked=clicked, + ), + ) response.status = 204 return {} diff --git a/optuna_dashboard/_custom_plot_data.py b/optuna_dashboard/_custom_plot_data.py new file mode 100644 index 00000000..a4dfa4af --- /dev/null +++ b/optuna_dashboard/_custom_plot_data.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import math +from typing import TYPE_CHECKING +import uuid + +from optuna import Study + + +if TYPE_CHECKING: + from typing import Any + + from optuna.storages import BaseStorage + import plotly.graph_objs as go + + +SYSTEM_ATTR_PLOT_DATA = "dashboard:plot_data:" +SYSTEM_ATTR_MAX_LENGTH = 2045 + + +def save_plotly_graph_object( + study: Study, figure: go.Figure, *, graph_object_id: str | None = None +) -> str: + """Save the user-defined plotly's graph object to the study. + + Example: + + .. code-block:: python + + import optuna + from optuna_dashboard import save_plotly_graph_object + + def objective(trial): + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + study.optimize(objective, n_trials=100) + + figure = optuna.visualization.plot_optimization_history(study) + save_plotly_graph_object(study, figure) + + Args: + study: + Target study object. + plot_data: + The plotly's graph object to save. + graph_object_id: + Unique identifier of the graph object. If specified, the graph object is overwritten. + This must be a valid HTML id attribute value. + + Returns: + The graph object ID. + """ + if graph_object_id is not None and not is_valid_graph_object_id(graph_object_id): + raise ValueError("graph_object_id must be a valid HTML id attribute value.") + + storage = study._storage + study_id = study._study_id + + graph_object_id = graph_object_id or str(uuid.uuid4()) + key = SYSTEM_ATTR_PLOT_DATA + graph_object_id + ":" + plot_data_json_str = figure.to_json() + save_graph_object_json(storage, study_id, key, plot_data_json_str) + return graph_object_id + + +def save_graph_object_json( + storage: BaseStorage, study_id: int, key_prefix: str, plot_data_json_str: str +) -> None: + plot_data_system_attrs = split_plot_data(plot_data_json_str, key_prefix) + for k, v in plot_data_system_attrs.items(): + storage.set_study_system_attr(study_id, k, v) + + # Clear previous graph object attributes + study_system_attrs = storage.get_study_system_attrs(study_id) + all_plot_data_system_attrs = [k for k in study_system_attrs if k.startswith(key_prefix)] + if len(all_plot_data_system_attrs) > len(plot_data_system_attrs): + for i in range(len(plot_data_system_attrs), len(all_plot_data_system_attrs)): + storage.set_study_system_attr(study_id, f"{key_prefix}{i}", "") + + +def list_graph_object_ids(system_attrs: dict[str, Any]) -> list[str]: + titles = set() + for key in system_attrs: + if not key.startswith(SYSTEM_ATTR_PLOT_DATA): + continue + + s = key.split(":", maxsplit=2) # e.g. ["dashboard", "plot_data", "Optimization History:1"] + if len(s) != 3: + continue + # Please note that title may contain ":". + title = s[2].rsplit(":", maxsplit=1)[0] + titles.add(title) + return list(titles) + + +def get_plotly_graph_objects(system_attrs: dict[str, Any]) -> dict[str, str]: + graph_objects = {} + for title in list_graph_object_ids(system_attrs): + key_prefix = SYSTEM_ATTR_PLOT_DATA + title + ":" + plot_data_attrs = {k: v for k, v in system_attrs.items() if k.startswith(key_prefix)} + graph_objects[title] = concat_plot_data(plot_data_attrs, key_prefix) + return graph_objects + + +def split_plot_data(plot_data_str: str, key_prefix: str) -> dict[str, str]: + plot_data_len = len(plot_data_str) + attrs = {} + for i in range(math.ceil(plot_data_len / SYSTEM_ATTR_MAX_LENGTH)): + start = i * SYSTEM_ATTR_MAX_LENGTH + end = min((i + 1) * SYSTEM_ATTR_MAX_LENGTH, plot_data_len) + attrs[f"{key_prefix}{i}"] = plot_data_str[start:end] + return attrs + + +def concat_plot_data(plot_data_attrs: dict[str, str], key_prefix: str) -> str: + return "".join(plot_data_attrs[f"{key_prefix}{i}"] for i in range(len(plot_data_attrs))) + + +def is_valid_graph_object_id(graph_object_id: str) -> bool: + if len(graph_object_id) == 0: + return False + + # Can only contain letters [A-Za-z], numbers [0-9], hyphens ("-"), underscores ("_"), + # colons, and periods. + if not all( + "a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9" or c in ("-", "_", ":", ".") + for c in graph_object_id[1:] + ): + return False + # Unlike HTML id attribute, graph object id can begin with a letter [A-Za-z] + return True diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py new file mode 100644 index 00000000..6b81b9bb --- /dev/null +++ b/optuna_dashboard/_preferential_history.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +import json +from typing import TYPE_CHECKING +import uuid + +from optuna.storages import BaseStorage + +from .preferential._system_attrs import report_preferences + + +_SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" + +if TYPE_CHECKING: + from typing import Literal + from typing import TypedDict + + FeedbackMode = Literal["ChooseWorst"] + ChooseWorstHistory = TypedDict( + "ChooseWorstHistory", + { + "mode": FeedbackMode, + "id": str, + "preference_id": str, + "timestamp": str, + "candidates": list[int], + "clicked": int, + }, + ) + History = ChooseWorstHistory + + +@dataclass +class NewHistory: + mode: FeedbackMode + candidates: list[int] + clicked: int + + +def report_history( + study_id: int, + storage: BaseStorage, + input_data: NewHistory, +) -> None: + preferences = [] + # TODO(moririn): Use TypeGuard after adding other history types. + if input_data.mode == "ChooseWorst": + preferences = [ + (best, input_data.clicked) + for best in input_data.candidates + if best != input_data.clicked + ] + else: + assert False, f"Unknown data: {input_data}" + + preference_id = report_preferences( + study_id=study_id, + storage=storage, + preferences=preferences, + ) + history_id = str(uuid.uuid4()) + + if input_data.mode == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": history_id, + "preference_id": preference_id, + "timestamp": datetime.now().isoformat(), + "candidates": input_data.candidates, + "clicked": input_data.clicked, + } + + key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id + storage.set_study_system_attr( + study_id=study_id, + key=key, + value=json.dumps(history), + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 0fa04124..19acbbd5 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import json from typing import Any from typing import TYPE_CHECKING @@ -14,6 +15,7 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names +from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -22,6 +24,9 @@ if TYPE_CHECKING: from typing import Literal from typing import TypedDict + from ._preferential_history import ChooseWorstHistory + from ._preferential_history import History + Attribute = TypedDict( "Attribute", { @@ -127,6 +132,7 @@ def serialize_study_detail( union: list[tuple[str, BaseDistribution]], union_user_attrs: list[tuple[str, bool]], has_intermediate_values: bool, + plotly_graph_objects: dict[str, str], ) -> dict[str, Any]: serialized: dict[str, Any] = { "name": summary.study_name, @@ -155,9 +161,38 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + if serialized["is_preferential"]: + serialized["preference_history"] = serialize_preference_history(system_attrs) + serialized["plotly_graph_objects"] = [ + {"id": id_, "graph_object": graph_object} + for id_, graph_object in plotly_graph_objects.items() + ] return serialized +def serialize_preference_history( + system_attrs: dict[str, Any], +) -> list[History]: + histories: list[History] = [] + for k, v in system_attrs.items(): + if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): + continue + choice: dict[str, Any] = json.loads(v) + if choice["mode"] == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": choice["id"], + "preference_id": choice["preference_id"], + "timestamp": choice["timestamp"], + "candidates": choice["candidates"], + "clicked": choice["clicked"], + } + histories.append(history) + + histories.sort(key=lambda c: datetime.fromisoformat(c["timestamp"])) + return histories + + def serialize_frozen_trial( study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any] ) -> dict[str, Any]: diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index e412d0ee..fb5411ee 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -174,6 +174,38 @@ class PreferentialStudy: """ self._study.add_trials(trials) + def enqueue_trial( + self, + params: dict[str, Any], + user_attrs: dict[str, Any] | None = None, + skip_if_exists: bool = False, + ) -> None: + """Enqueue a trial with given parameter values. + + You can fix the next sampling parameters which will be evaluated in your + objective function. + + .. seealso:: + + See `Study.enqueue_trials`_ for details. + + .. _Study.get_trials: https://optuna.readthedocs.io/en/stable/reference/\ + generated/optuna.study.Study.html#optuna.study.Study.enqueue_trials + + Args: + params: + Parameter values to pass your objective function. + user_attrs: + A dictionary of user-specific attributes other than ``params``. + skip_if_exists: + When :obj:`True`, prevents duplicate trials from being enqueued again. + + .. note:: + This method might produce duplicated trials if called simultaneously + by multiple processes at the same time with same ``params`` dict. + """ + self._study.enqueue_trial(params, user_attrs, skip_if_exists) + def report_preference( self, better_trials: FrozenTrial | list[FrozenTrial], diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 2964c9e0..4cb0e288 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -16,8 +16,9 @@ def report_preferences( study_id: int, storage: BaseStorage, preferences: list[tuple[int, int]], -) -> None: - key = _SYSTEM_ATTR_PREFIX_PREFERENCE + str(uuid.uuid4()) +) -> str: + preference_id = str(uuid.uuid4()) + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_id storage.set_study_system_attr( study_id=study_id, key=key, @@ -31,6 +32,7 @@ def report_preferences( trial_id = trials[number]._trial_id if trials[number].state != TrialState.COMPLETE: storage.set_trial_state_values(trial_id, TrialState.COMPLETE, values) + return preference_id def get_preferences( diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index 017751fa..e1896178 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -587,10 +587,10 @@ export const actionCreator = () => { const updatePreference = ( study_id: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ) => { - reportPreferenceAPI(study_id, best_trials, worst_trials).catch((err) => { + reportPreferenceAPI(study_id, candidates, clicked).catch((err) => { const reason = err.response?.data.reason enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { variant: "error", diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 25ca3541..e62e0e42 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -55,6 +55,28 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } +interface PreferenceHistoryResponce { + id: string + preference_id: string + candidates: number[] + clicked: number + mode: PreferenceFeedbackMode + timestamp: string +} + +const convertPreferenceHistory = ( + res: PreferenceHistoryResponce +): PreferenceHistory => { + return { + id: res.id, + preference_id: res.preference_id, + candidates: res.candidates, + clicked: res.clicked, + feedback_mode: res.mode, + timestamp: new Date(res.timestamp), + } +} + interface StudyDetailResponse { name: string datetime_start: string @@ -70,6 +92,8 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + preference_history?: PreferenceHistoryResponce[] + plotly_graph_objects: PlotlyGraphObject[] } export const getStudyDetailAPI = ( @@ -105,6 +129,10 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, + preference_history: res.data.preference_history?.map( + convertPreferenceHistory + ), + plotly_graph_objects: res.data.plotly_graph_objects, } }) } @@ -314,13 +342,14 @@ export const getParamImportances = ( export const reportPreferenceAPI = ( studyId: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ): Promise => { return axiosInstance .post(`/api/studies/${studyId}/preference`, { - best_trials: best_trials, - worst_trials: worst_trials, + candidates: candidates, + clicked: clicked, + mode: "ChooseWorst", }) .then(() => { return diff --git a/optuna_dashboard/ts/components/App.tsx b/optuna_dashboard/ts/components/App.tsx index 78015049..8adf8895 100644 --- a/optuna_dashboard/ts/components/App.tsx +++ b/optuna_dashboard/ts/components/App.tsx @@ -96,6 +96,15 @@ export const App: FC = () => { /> } /> + + } + /> } diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 0903d72b..446b362e 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -34,12 +34,19 @@ import GitHubIcon from "@mui/icons-material/GitHub" import OpenInNewIcon from "@mui/icons-material/OpenInNew" import QueryStatsIcon from "@mui/icons-material/QueryStats" import ThumbUpAltIcon from "@mui/icons-material/ThumbUpAlt" +import HistoryIcon from "@mui/icons-material/History" import { Switch } from "@mui/material" import { actionCreator } from "../action" const drawerWidth = 240 -export type PageId = "top" | "analytics" | "trialTable" | "trialList" | "note" +export type PageId = + | "top" + | "analytics" + | "trialTable" + | "trialList" + | "note" + | "preferenceHistory" const openedMixin = (theme: Theme): CSSObject => ({ width: drawerWidth, @@ -204,6 +211,28 @@ export const AppDrawer: FC<{ /> + {isPreferential && ( + + + + + + + + + )} = ({ trial, type }) => { + const theme = useTheme() + const trialWidth = 300 + const trialHeight = 300 + const [detailShown, setDetailShown] = useState(false) + + const cardComponentSx = { + padding: 0, + position: "relative", + overflow: "hidden", + "::before": {}, + } + if (type !== "none") { + cardComponentSx["::before"] = { + content: '""', + position: "absolute", + top: 0, + left: 0, + width: "100%", + height: "100%", + backgroundColor: theme.palette.mode === "dark" ? "white" : "black", + opacity: 0.2, + zIndex: 1, + transition: "opacity 0.3s ease-out", + } + } + + return ( + + + Trial {trial.number} + setDetailShown(true)} + aria-label="show detail" + > + + + + + + + + + {type === "worst" ? ( + + ) : null} + + setDetailShown(false)}> + + + false} + directions={[]} + objectiveNames={[]} + /> + + + + + ) +} + +const ChoiceTrials: FC<{ choice: PreferenceHistory; trials: Trial[] }> = ({ + choice, + trials, +}) => { + const theme = useTheme() + const worst_trials = new Set([choice.clicked]) + + return ( + + + {formatDate(choice.timestamp)} + + + {choice.candidates.map((trial_num, index) => ( + + ))} + + + ) +} + +export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({ + studyDetail, +}) => { + if ( + studyDetail === null || + !studyDetail.is_preferential || + studyDetail.preference_history === undefined + ) { + return null + } + const theme = useTheme() + const preference_histories = [...studyDetail.preference_history] + + if (preference_histories.length === 0) { + return ( + + No feedback history + + ) + } + + return ( + + {preference_histories.reverse().map((choice) => ( + + ))} + + ) +} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 5c141403..93ed2f5a 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -21,9 +21,9 @@ import { MarkdownRenderer } from "./Note" const PreferentialTrial: FC<{ trial?: Trial - studyDetail: StudyDetail + candidates: number[] hideTrial: () => void -}> = ({ trial, studyDetail, hideTrial }) => { +}> = ({ trial, candidates, hideTrial }) => { const theme = useTheme() const action = actionCreator() const trialWidth = 500 @@ -80,10 +80,7 @@ const PreferentialTrial: FC<{ aria-label="trial-button" onClick={() => { hideTrial() - const best_trials = studyDetail.best_trials - .map((t) => t.number) - .filter((t) => t !== trial.number) - action.updatePreference(trial.study_id, best_trials, [trial.number]) + action.updatePreference(trial.study_id, candidates, trial.number) }} sx={{ padding: 0, @@ -243,7 +240,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ trial.number === t)} - studyDetail={studyDetail} + candidates={displayTrials.numbers.filter((n) => n !== -1)} hideTrial={() => { hideTrial(t) }} diff --git a/optuna_dashboard/ts/components/StudyDetail.tsx b/optuna_dashboard/ts/components/StudyDetail.tsx index fedfd625..15cb7cf8 100644 --- a/optuna_dashboard/ts/components/StudyDetail.tsx +++ b/optuna_dashboard/ts/components/StudyDetail.tsx @@ -30,6 +30,7 @@ import { GraphEdf } from "./GraphEdf" import { TrialList } from "./TrialList" import { StudyHistory } from "./StudyHistory" import { PreferentialTrials } from "./PreferentialTrials" +import { PreferenceHistory } from "./PreferenceHistory" import { PreferentialAnalytics } from "./PreferentialAnalytics" interface ParamTypes { @@ -175,6 +176,8 @@ export const StudyDetail: FC<{ /> ) + } else if (page == "preferenceHistory") { + content = } const toolbar = ( diff --git a/optuna_dashboard/ts/components/StudyHistory.tsx b/optuna_dashboard/ts/components/StudyHistory.tsx index b47c557a..907acd1a 100644 --- a/optuna_dashboard/ts/components/StudyHistory.tsx +++ b/optuna_dashboard/ts/components/StudyHistory.tsx @@ -15,6 +15,7 @@ import { GraphIntermediateValues } from "./GraphIntermediateValues" import Grid2 from "@mui/material/Unstable_Grid2" import { DataGrid, DataGridColumn } from "./DataGrid" import { GraphHyperparameterImportance } from "./GraphHyperparameterImportances" +import { UserDefinedPlot } from "./UserDefinedPlot" import { BestTrialsCard } from "./BestTrialsCard" import { useStudyDetailValue, @@ -124,6 +125,16 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => { + {studyDetail !== null && + studyDetail.plotly_graph_objects.map((go) => ( + + + + + + + + ))} diff --git a/optuna_dashboard/ts/components/UserDefinedPlot.tsx b/optuna_dashboard/ts/components/UserDefinedPlot.tsx new file mode 100644 index 00000000..029c4b57 --- /dev/null +++ b/optuna_dashboard/ts/components/UserDefinedPlot.tsx @@ -0,0 +1,21 @@ +import * as plotly from "plotly.js-dist-min" +import React, { FC, useEffect } from "react" +import { Box } from "@mui/material" + +export const UserDefinedPlot: FC<{ + graphObject: PlotlyGraphObject +}> = ({ graphObject }) => { + const plotDomId = `user-defined-plot:${graphObject.id}` + + useEffect(() => { + try { + const parsed = JSON.parse(graphObject.graph_object) + plotly.react(plotDomId, parsed.data, parsed.layout) + } catch (e) { + // Avoid to crash the whole page when given invalid grpah objects. + console.error(e) + } + }, [graphObject]) + + return +} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 1720cc6b..b7b35797 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -12,6 +12,7 @@ type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type TrialStateFinished = "Complete" | "Fail" | "Pruned" type StudyDirection = "maximize" | "minimize" | "not_set" +type PreferenceFeedbackMode = "ChooseWorst" type FloatDistribution = { type: "FloatDistribution" @@ -181,6 +182,11 @@ type FormWidgets = widgets: UserAttrFormWidget[] } +type PlotlyGraphObject = { + id: string + graph_object: string +} + type StudyDetail = { id: number name: string @@ -197,6 +203,8 @@ type StudyDetail = { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + preference_history?: PreferenceHistory[] + plotly_graph_objects: PlotlyGraphObject[] } type StudyDetails = { @@ -206,3 +214,12 @@ type StudyDetails = { type StudyParamImportance = { [study_id: string]: ParamImportance[][] } + +type PreferenceHistory = { + id: string + preference_id: string + candidates: number[] + clicked: number + feedback_mode: PreferenceFeedbackMode + timestamp: Date +} diff --git a/pyproject.toml b/pyproject.toml index 0555f701..9b7ce731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ docs = [ test = [ "coverage", + "plotly", "pytest", "moto[s3]", ] diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 5085b598..75243f30 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -135,7 +135,13 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference", "POST", - body=json.dumps({"best_trials": [0, 2], "worst_trials": [1]}), + body=json.dumps( + { + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), content_type="application/json", ) self.assertEqual(status, 204) @@ -150,6 +156,30 @@ class APITestCase(TestCase): assert better.number == 2 assert worse.number == 1 + def test_report_preference_when_typo_mode(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + for _ in range(3): + trial = study.ask() + study.mark_comparison_ready(trial) + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference", + "POST", + body=json.dumps( + { + "mode": "ChoseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 400) + def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) diff --git a/python_tests/test_custom_plot_data.py b/python_tests/test_custom_plot_data.py new file mode 100644 index 00000000..3dcfc856 --- /dev/null +++ b/python_tests/test_custom_plot_data.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import optuna +from optuna_dashboard import _custom_plot_data as custom_plot_data +from optuna_dashboard import save_plotly_graph_object +import pytest + + +def get_dummy_study() -> optuna.Study: + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + optuna.logging.set_verbosity(optuna.logging.ERROR) + study.optimize(objective, n_trials=100) + return study + + +def test_save_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 2 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +def test_update_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object( + dummy_study, plot_data, graph_object_id=graph_object_id + ) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +@pytest.mark.parametrize( + "name", + [ + "0", + "a", + "a1-:_.", + ], +) +def test_is_valid_graph_object_id(name: str) -> None: + assert custom_plot_data.is_valid_graph_object_id(name) + + +@pytest.mark.parametrize( + "name", + [ + "a,", + "a b", + "aあいうえお", + ], +) +def test_is_invalid_graph_object_id(name: str) -> None: + assert not custom_plot_data.is_valid_graph_object_id(name) diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py new file mode 100644 index 00000000..ab524b90 --- /dev/null +++ b/python_tests/test_preferential_history.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from typing import Callable + +from optuna_dashboard._preferential_history import NewHistory +from optuna_dashboard._preferential_history import report_history +from optuna_dashboard._serializer import serialize_preference_history +from optuna_dashboard.preferential import create_study +from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE + +from .storage_supplier import parametrize_storages +from .storage_supplier import StorageSupplier + + +@parametrize_storages +def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage, n_generate=5) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) + study.mark_comparison_ready(trial) + + study_id = study._study._study_id + + report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 1, 2], + clicked=1, + ), + ) + report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 2, 3, 4], + clicked=0, + ), + ) + history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + sys_attrs = storage.get_study_system_attrs(study_id) + assert len(history) == 2 + assert history[0]["candidates"] == [0, 1, 2] + assert history[0]["clicked"] == 1 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]] + assert len(preferences) == 2 + for i, (best, worst) in enumerate([(0, 1), (2, 1)]): + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst + assert history[1]["candidates"] == [0, 2, 3, 4] + assert history[1]["clicked"] == 0 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]] + assert len(preferences) == 3 + for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index a90e0de7..72db7b26 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -29,7 +29,7 @@ def test_get_study_detail_is_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert study_detail["is_preferential"] @@ -40,7 +40,7 @@ def test_get_study_detail_is_not_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert not study_detail["is_preferential"]