mirror of
https://github.com/wassname/optuna-dashboard.git
synced 2026-09-10 12:23:22 +08:00
Merge branch 'main' of github.com:optuna/optuna-dashboard into active_trials
This commit is contained in:
Contramundum
@@ -45,4 +45,4 @@ jobs:
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
file: ./coverage.xml
|
||||
fail_ci_if_error: true
|
||||
fail_ci_if_error: false
|
||||
|
||||
@@ -14,6 +14,7 @@ General APIs
|
||||
optuna_dashboard.wsgi
|
||||
optuna_dashboard.set_objective_names
|
||||
optuna_dashboard.save_note
|
||||
optuna_dashboard.save_plotly_graph_object
|
||||
|
||||
Human-in-the-loop
|
||||
-----------------
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from ._app import run_server # noqa
|
||||
from ._app import wsgi # noqa
|
||||
from ._custom_plot_data import save_plotly_graph_object # noqa
|
||||
from ._form_widget import ChoiceWidget # noqa
|
||||
from ._form_widget import dict_to_form_widget # noqa
|
||||
from ._form_widget import ObjectiveChoiceWidget # noqa
|
||||
@@ -15,4 +16,4 @@ from ._note import get_note # noqa
|
||||
from ._note import save_note # noqa
|
||||
|
||||
|
||||
__version__ = "0.12.0"
|
||||
__version__ = "0.13.0b1"
|
||||
|
||||
@@ -25,8 +25,11 @@ from . import _note as note
|
||||
from ._bottle_util import BottleViewReturn
|
||||
from ._bottle_util import json_api_view
|
||||
from ._cached_extra_study_property import get_cached_extra_study_property
|
||||
from ._custom_plot_data import get_plotly_graph_objects
|
||||
from ._importance import get_param_importance_from_trials_cache
|
||||
from ._pareto_front import get_pareto_front_trials
|
||||
from ._preferential_history import NewHistory
|
||||
from ._preferential_history import report_history
|
||||
from ._rdb_migration import register_rdb_migration_route
|
||||
from ._serializer import serialize_study_detail
|
||||
from ._serializer import serialize_study_summary
|
||||
@@ -40,7 +43,6 @@ from .artifact._backend import register_artifact_route
|
||||
from .artifact._backend_to_store import to_artifact_store
|
||||
from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY
|
||||
from .preferential._study import get_best_trials as get_best_preferential_trials
|
||||
from .preferential._system_attrs import report_preferences
|
||||
from .preferential._system_attrs import report_skip
|
||||
|
||||
|
||||
@@ -213,6 +215,8 @@ def create_app(
|
||||
union_user_attrs,
|
||||
has_intermediate_values,
|
||||
) = get_cached_extra_study_property(study_id, trials)
|
||||
|
||||
plotly_graph_objects = get_plotly_graph_objects(system_attrs)
|
||||
return serialize_study_detail(
|
||||
summary,
|
||||
best_trials,
|
||||
@@ -221,6 +225,7 @@ def create_app(
|
||||
union,
|
||||
union_user_attrs,
|
||||
has_intermediate_values,
|
||||
plotly_graph_objects,
|
||||
)
|
||||
|
||||
@app.get("/api/studies/<study_id:int>/param_importances")
|
||||
@@ -269,17 +274,34 @@ def create_app(
|
||||
@json_api_view
|
||||
def post_preference(study_id: int) -> dict[str, Any]:
|
||||
try:
|
||||
best_trials = [int(d) for d in request.json.get("best_trials", [])]
|
||||
worst_trials = [int(d) for d in request.json.get("worst_trials", [])]
|
||||
mode = request.json.get("mode", "")
|
||||
candidates = [int(d) for d in request.json.get("candidates", [])]
|
||||
clicked = int(request.json.get("clicked", -1))
|
||||
except ValueError:
|
||||
response.status = 400
|
||||
return {"reason": "best_trials and worst_trials must be an array of integers."}
|
||||
if len(best_trials) == 0 or len(worst_trials) == 0:
|
||||
response.status = 400 # Bad request
|
||||
return {"reason": "You need to set best_trials and worst_trials"}
|
||||
return {
|
||||
"reason": (
|
||||
"`candidates` should be an array of integers and "
|
||||
"`clicked` should be an integer."
|
||||
)
|
||||
}
|
||||
|
||||
preferences = [(best, worst) for best in best_trials for worst in worst_trials]
|
||||
report_preferences(study_id, storage, preferences)
|
||||
if clicked == -1:
|
||||
response.status = 400
|
||||
return {"reason": "`clicked` should be specified."}
|
||||
if mode != "ChooseWorst":
|
||||
response.status = 400
|
||||
return {"reason": "`mode` should be 'ChooseWorst'."}
|
||||
|
||||
report_history(
|
||||
study_id,
|
||||
storage,
|
||||
NewHistory(
|
||||
mode=mode,
|
||||
candidates=candidates,
|
||||
clicked=clicked,
|
||||
),
|
||||
)
|
||||
|
||||
response.status = 204
|
||||
return {}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import TYPE_CHECKING
|
||||
import uuid
|
||||
|
||||
from optuna import Study
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Any
|
||||
|
||||
from optuna.storages import BaseStorage
|
||||
import plotly.graph_objs as go
|
||||
|
||||
|
||||
SYSTEM_ATTR_PLOT_DATA = "dashboard:plot_data:"
|
||||
SYSTEM_ATTR_MAX_LENGTH = 2045
|
||||
|
||||
|
||||
def save_plotly_graph_object(
|
||||
study: Study, figure: go.Figure, *, graph_object_id: str | None = None
|
||||
) -> str:
|
||||
"""Save the user-defined plotly's graph object to the study.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import optuna
|
||||
from optuna_dashboard import save_plotly_graph_object
|
||||
|
||||
def objective(trial):
|
||||
x = trial.suggest_float("x", -100, 100)
|
||||
y = trial.suggest_categorical("y", [-1, 0, 1])
|
||||
return x**2 + y
|
||||
|
||||
study = optuna.create_study()
|
||||
study.optimize(objective, n_trials=100)
|
||||
|
||||
figure = optuna.visualization.plot_optimization_history(study)
|
||||
save_plotly_graph_object(study, figure)
|
||||
|
||||
Args:
|
||||
study:
|
||||
Target study object.
|
||||
plot_data:
|
||||
The plotly's graph object to save.
|
||||
graph_object_id:
|
||||
Unique identifier of the graph object. If specified, the graph object is overwritten.
|
||||
This must be a valid HTML id attribute value.
|
||||
|
||||
Returns:
|
||||
The graph object ID.
|
||||
"""
|
||||
if graph_object_id is not None and not is_valid_graph_object_id(graph_object_id):
|
||||
raise ValueError("graph_object_id must be a valid HTML id attribute value.")
|
||||
|
||||
storage = study._storage
|
||||
study_id = study._study_id
|
||||
|
||||
graph_object_id = graph_object_id or str(uuid.uuid4())
|
||||
key = SYSTEM_ATTR_PLOT_DATA + graph_object_id + ":"
|
||||
plot_data_json_str = figure.to_json()
|
||||
save_graph_object_json(storage, study_id, key, plot_data_json_str)
|
||||
return graph_object_id
|
||||
|
||||
|
||||
def save_graph_object_json(
|
||||
storage: BaseStorage, study_id: int, key_prefix: str, plot_data_json_str: str
|
||||
) -> None:
|
||||
plot_data_system_attrs = split_plot_data(plot_data_json_str, key_prefix)
|
||||
for k, v in plot_data_system_attrs.items():
|
||||
storage.set_study_system_attr(study_id, k, v)
|
||||
|
||||
# Clear previous graph object attributes
|
||||
study_system_attrs = storage.get_study_system_attrs(study_id)
|
||||
all_plot_data_system_attrs = [k for k in study_system_attrs if k.startswith(key_prefix)]
|
||||
if len(all_plot_data_system_attrs) > len(plot_data_system_attrs):
|
||||
for i in range(len(plot_data_system_attrs), len(all_plot_data_system_attrs)):
|
||||
storage.set_study_system_attr(study_id, f"{key_prefix}{i}", "")
|
||||
|
||||
|
||||
def list_graph_object_ids(system_attrs: dict[str, Any]) -> list[str]:
|
||||
titles = set()
|
||||
for key in system_attrs:
|
||||
if not key.startswith(SYSTEM_ATTR_PLOT_DATA):
|
||||
continue
|
||||
|
||||
s = key.split(":", maxsplit=2) # e.g. ["dashboard", "plot_data", "Optimization History:1"]
|
||||
if len(s) != 3:
|
||||
continue
|
||||
# Please note that title may contain ":".
|
||||
title = s[2].rsplit(":", maxsplit=1)[0]
|
||||
titles.add(title)
|
||||
return list(titles)
|
||||
|
||||
|
||||
def get_plotly_graph_objects(system_attrs: dict[str, Any]) -> dict[str, str]:
|
||||
graph_objects = {}
|
||||
for title in list_graph_object_ids(system_attrs):
|
||||
key_prefix = SYSTEM_ATTR_PLOT_DATA + title + ":"
|
||||
plot_data_attrs = {k: v for k, v in system_attrs.items() if k.startswith(key_prefix)}
|
||||
graph_objects[title] = concat_plot_data(plot_data_attrs, key_prefix)
|
||||
return graph_objects
|
||||
|
||||
|
||||
def split_plot_data(plot_data_str: str, key_prefix: str) -> dict[str, str]:
|
||||
plot_data_len = len(plot_data_str)
|
||||
attrs = {}
|
||||
for i in range(math.ceil(plot_data_len / SYSTEM_ATTR_MAX_LENGTH)):
|
||||
start = i * SYSTEM_ATTR_MAX_LENGTH
|
||||
end = min((i + 1) * SYSTEM_ATTR_MAX_LENGTH, plot_data_len)
|
||||
attrs[f"{key_prefix}{i}"] = plot_data_str[start:end]
|
||||
return attrs
|
||||
|
||||
|
||||
def concat_plot_data(plot_data_attrs: dict[str, str], key_prefix: str) -> str:
|
||||
return "".join(plot_data_attrs[f"{key_prefix}{i}"] for i in range(len(plot_data_attrs)))
|
||||
|
||||
|
||||
def is_valid_graph_object_id(graph_object_id: str) -> bool:
|
||||
if len(graph_object_id) == 0:
|
||||
return False
|
||||
|
||||
# Can only contain letters [A-Za-z], numbers [0-9], hyphens ("-"), underscores ("_"),
|
||||
# colons, and periods.
|
||||
if not all(
|
||||
"a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9" or c in ("-", "_", ":", ".")
|
||||
for c in graph_object_id[1:]
|
||||
):
|
||||
return False
|
||||
# Unlike HTML id attribute, graph object id can begin with a letter [A-Za-z]
|
||||
return True
|
||||
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
import uuid
|
||||
|
||||
from optuna.storages import BaseStorage
|
||||
|
||||
from .preferential._system_attrs import report_preferences
|
||||
|
||||
|
||||
_SYSTEM_ATTR_PREFIX_HISTORY = "preference:history"
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Literal
|
||||
from typing import TypedDict
|
||||
|
||||
FeedbackMode = Literal["ChooseWorst"]
|
||||
ChooseWorstHistory = TypedDict(
|
||||
"ChooseWorstHistory",
|
||||
{
|
||||
"mode": FeedbackMode,
|
||||
"id": str,
|
||||
"preference_id": str,
|
||||
"timestamp": str,
|
||||
"candidates": list[int],
|
||||
"clicked": int,
|
||||
},
|
||||
)
|
||||
History = ChooseWorstHistory
|
||||
|
||||
|
||||
@dataclass
|
||||
class NewHistory:
|
||||
mode: FeedbackMode
|
||||
candidates: list[int]
|
||||
clicked: int
|
||||
|
||||
|
||||
def report_history(
|
||||
study_id: int,
|
||||
storage: BaseStorage,
|
||||
input_data: NewHistory,
|
||||
) -> None:
|
||||
preferences = []
|
||||
# TODO(moririn): Use TypeGuard after adding other history types.
|
||||
if input_data.mode == "ChooseWorst":
|
||||
preferences = [
|
||||
(best, input_data.clicked)
|
||||
for best in input_data.candidates
|
||||
if best != input_data.clicked
|
||||
]
|
||||
else:
|
||||
assert False, f"Unknown data: {input_data}"
|
||||
|
||||
preference_id = report_preferences(
|
||||
study_id=study_id,
|
||||
storage=storage,
|
||||
preferences=preferences,
|
||||
)
|
||||
history_id = str(uuid.uuid4())
|
||||
|
||||
if input_data.mode == "ChooseWorst":
|
||||
history: ChooseWorstHistory = {
|
||||
"mode": "ChooseWorst",
|
||||
"id": history_id,
|
||||
"preference_id": preference_id,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"candidates": input_data.candidates,
|
||||
"clicked": input_data.clicked,
|
||||
}
|
||||
|
||||
key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id
|
||||
storage.set_study_system_attr(
|
||||
study_id=study_id,
|
||||
key=key,
|
||||
value=json.dumps(history),
|
||||
)
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
import json
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -14,6 +15,7 @@ from optuna.trial import FrozenTrial
|
||||
from . import _note as note
|
||||
from ._form_widget import get_form_widgets_json
|
||||
from ._named_objectives import get_objective_names
|
||||
from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY
|
||||
from .artifact._backend import list_trial_artifacts
|
||||
from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY
|
||||
|
||||
@@ -22,6 +24,9 @@ if TYPE_CHECKING:
|
||||
from typing import Literal
|
||||
from typing import TypedDict
|
||||
|
||||
from ._preferential_history import ChooseWorstHistory
|
||||
from ._preferential_history import History
|
||||
|
||||
Attribute = TypedDict(
|
||||
"Attribute",
|
||||
{
|
||||
@@ -127,6 +132,7 @@ def serialize_study_detail(
|
||||
union: list[tuple[str, BaseDistribution]],
|
||||
union_user_attrs: list[tuple[str, bool]],
|
||||
has_intermediate_values: bool,
|
||||
plotly_graph_objects: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
serialized: dict[str, Any] = {
|
||||
"name": summary.study_name,
|
||||
@@ -155,9 +161,38 @@ def serialize_study_detail(
|
||||
form_widgets = get_form_widgets_json(system_attrs)
|
||||
if form_widgets:
|
||||
serialized["form_widgets"] = form_widgets
|
||||
if serialized["is_preferential"]:
|
||||
serialized["preference_history"] = serialize_preference_history(system_attrs)
|
||||
serialized["plotly_graph_objects"] = [
|
||||
{"id": id_, "graph_object": graph_object}
|
||||
for id_, graph_object in plotly_graph_objects.items()
|
||||
]
|
||||
return serialized
|
||||
|
||||
|
||||
def serialize_preference_history(
|
||||
system_attrs: dict[str, Any],
|
||||
) -> list[History]:
|
||||
histories: list[History] = []
|
||||
for k, v in system_attrs.items():
|
||||
if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY):
|
||||
continue
|
||||
choice: dict[str, Any] = json.loads(v)
|
||||
if choice["mode"] == "ChooseWorst":
|
||||
history: ChooseWorstHistory = {
|
||||
"mode": "ChooseWorst",
|
||||
"id": choice["id"],
|
||||
"preference_id": choice["preference_id"],
|
||||
"timestamp": choice["timestamp"],
|
||||
"candidates": choice["candidates"],
|
||||
"clicked": choice["clicked"],
|
||||
}
|
||||
histories.append(history)
|
||||
|
||||
histories.sort(key=lambda c: datetime.fromisoformat(c["timestamp"]))
|
||||
return histories
|
||||
|
||||
|
||||
def serialize_frozen_trial(
|
||||
study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
|
||||
@@ -174,6 +174,38 @@ class PreferentialStudy:
|
||||
"""
|
||||
self._study.add_trials(trials)
|
||||
|
||||
def enqueue_trial(
|
||||
self,
|
||||
params: dict[str, Any],
|
||||
user_attrs: dict[str, Any] | None = None,
|
||||
skip_if_exists: bool = False,
|
||||
) -> None:
|
||||
"""Enqueue a trial with given parameter values.
|
||||
|
||||
You can fix the next sampling parameters which will be evaluated in your
|
||||
objective function.
|
||||
|
||||
.. seealso::
|
||||
|
||||
See `Study.enqueue_trials`_ for details.
|
||||
|
||||
.. _Study.get_trials: https://optuna.readthedocs.io/en/stable/reference/\
|
||||
generated/optuna.study.Study.html#optuna.study.Study.enqueue_trials
|
||||
|
||||
Args:
|
||||
params:
|
||||
Parameter values to pass your objective function.
|
||||
user_attrs:
|
||||
A dictionary of user-specific attributes other than ``params``.
|
||||
skip_if_exists:
|
||||
When :obj:`True`, prevents duplicate trials from being enqueued again.
|
||||
|
||||
.. note::
|
||||
This method might produce duplicated trials if called simultaneously
|
||||
by multiple processes at the same time with same ``params`` dict.
|
||||
"""
|
||||
self._study.enqueue_trial(params, user_attrs, skip_if_exists)
|
||||
|
||||
def report_preference(
|
||||
self,
|
||||
better_trials: FrozenTrial | list[FrozenTrial],
|
||||
|
||||
@@ -16,8 +16,9 @@ def report_preferences(
|
||||
study_id: int,
|
||||
storage: BaseStorage,
|
||||
preferences: list[tuple[int, int]],
|
||||
) -> None:
|
||||
key = _SYSTEM_ATTR_PREFIX_PREFERENCE + str(uuid.uuid4())
|
||||
) -> str:
|
||||
preference_id = str(uuid.uuid4())
|
||||
key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_id
|
||||
storage.set_study_system_attr(
|
||||
study_id=study_id,
|
||||
key=key,
|
||||
@@ -31,6 +32,7 @@ def report_preferences(
|
||||
trial_id = trials[number]._trial_id
|
||||
if trials[number].state != TrialState.COMPLETE:
|
||||
storage.set_trial_state_values(trial_id, TrialState.COMPLETE, values)
|
||||
return preference_id
|
||||
|
||||
|
||||
def get_preferences(
|
||||
|
||||
@@ -587,10 +587,10 @@ export const actionCreator = () => {
|
||||
|
||||
const updatePreference = (
|
||||
study_id: number,
|
||||
best_trials: number[],
|
||||
worst_trials: number[]
|
||||
candidates: number[],
|
||||
clicked: number
|
||||
) => {
|
||||
reportPreferenceAPI(study_id, best_trials, worst_trials).catch((err) => {
|
||||
reportPreferenceAPI(study_id, candidates, clicked).catch((err) => {
|
||||
const reason = err.response?.data.reason
|
||||
enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, {
|
||||
variant: "error",
|
||||
|
||||
@@ -55,6 +55,28 @@ const convertTrialResponse = (res: TrialResponse): Trial => {
|
||||
}
|
||||
}
|
||||
|
||||
interface PreferenceHistoryResponce {
|
||||
id: string
|
||||
preference_id: string
|
||||
candidates: number[]
|
||||
clicked: number
|
||||
mode: PreferenceFeedbackMode
|
||||
timestamp: string
|
||||
}
|
||||
|
||||
const convertPreferenceHistory = (
|
||||
res: PreferenceHistoryResponce
|
||||
): PreferenceHistory => {
|
||||
return {
|
||||
id: res.id,
|
||||
preference_id: res.preference_id,
|
||||
candidates: res.candidates,
|
||||
clicked: res.clicked,
|
||||
feedback_mode: res.mode,
|
||||
timestamp: new Date(res.timestamp),
|
||||
}
|
||||
}
|
||||
|
||||
interface StudyDetailResponse {
|
||||
name: string
|
||||
datetime_start: string
|
||||
@@ -70,6 +92,8 @@ interface StudyDetailResponse {
|
||||
is_preferential: boolean
|
||||
objective_names?: string[]
|
||||
form_widgets?: FormWidgets
|
||||
preference_history?: PreferenceHistoryResponce[]
|
||||
plotly_graph_objects: PlotlyGraphObject[]
|
||||
}
|
||||
|
||||
export const getStudyDetailAPI = (
|
||||
@@ -105,6 +129,10 @@ export const getStudyDetailAPI = (
|
||||
objective_names: res.data.objective_names,
|
||||
form_widgets: res.data.form_widgets,
|
||||
is_preferential: res.data.is_preferential,
|
||||
preference_history: res.data.preference_history?.map(
|
||||
convertPreferenceHistory
|
||||
),
|
||||
plotly_graph_objects: res.data.plotly_graph_objects,
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -314,13 +342,14 @@ export const getParamImportances = (
|
||||
|
||||
export const reportPreferenceAPI = (
|
||||
studyId: number,
|
||||
best_trials: number[],
|
||||
worst_trials: number[]
|
||||
candidates: number[],
|
||||
clicked: number
|
||||
): Promise<void> => {
|
||||
return axiosInstance
|
||||
.post<void>(`/api/studies/${studyId}/preference`, {
|
||||
best_trials: best_trials,
|
||||
worst_trials: worst_trials,
|
||||
candidates: candidates,
|
||||
clicked: clicked,
|
||||
mode: "ChooseWorst",
|
||||
})
|
||||
.then(() => {
|
||||
return
|
||||
|
||||
@@ -96,6 +96,15 @@ export const App: FC = () => {
|
||||
/>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path={URL_PREFIX + "/studies/:studyId/preference-history"}
|
||||
element={
|
||||
<StudyDetail
|
||||
toggleColorMode={toggleColorMode}
|
||||
page={"preferenceHistory"}
|
||||
/>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path={URL_PREFIX + "/compare-studies"}
|
||||
element={<CompareStudies toggleColorMode={toggleColorMode} />}
|
||||
|
||||
@@ -34,12 +34,19 @@ import GitHubIcon from "@mui/icons-material/GitHub"
|
||||
import OpenInNewIcon from "@mui/icons-material/OpenInNew"
|
||||
import QueryStatsIcon from "@mui/icons-material/QueryStats"
|
||||
import ThumbUpAltIcon from "@mui/icons-material/ThumbUpAlt"
|
||||
import HistoryIcon from "@mui/icons-material/History"
|
||||
import { Switch } from "@mui/material"
|
||||
import { actionCreator } from "../action"
|
||||
|
||||
const drawerWidth = 240
|
||||
|
||||
export type PageId = "top" | "analytics" | "trialTable" | "trialList" | "note"
|
||||
export type PageId =
|
||||
| "top"
|
||||
| "analytics"
|
||||
| "trialTable"
|
||||
| "trialList"
|
||||
| "note"
|
||||
| "preferenceHistory"
|
||||
|
||||
const openedMixin = (theme: Theme): CSSObject => ({
|
||||
width: drawerWidth,
|
||||
@@ -204,6 +211,28 @@ export const AppDrawer: FC<{
|
||||
/>
|
||||
</ListItemButton>
|
||||
</ListItem>
|
||||
{isPreferential && (
|
||||
<ListItem
|
||||
key="PreferenceHistory"
|
||||
disablePadding
|
||||
sx={styleListItem}
|
||||
>
|
||||
<ListItemButton
|
||||
component={Link}
|
||||
to={`${URL_PREFIX}/studies/${studyId}/preference-history`}
|
||||
sx={styleListItemButton}
|
||||
selected={page === "preferenceHistory"}
|
||||
>
|
||||
<ListItemIcon sx={styleListItemIcon}>
|
||||
<HistoryIcon />
|
||||
</ListItemIcon>
|
||||
<ListItemText
|
||||
primary="PreferenceHistory"
|
||||
sx={styleListItemText}
|
||||
/>
|
||||
</ListItemButton>
|
||||
</ListItem>
|
||||
)}
|
||||
<ListItem key="Analytics" disablePadding sx={styleListItem}>
|
||||
<ListItemButton
|
||||
component={Link}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
import React, { FC, useState } from "react"
|
||||
import {
|
||||
Typography,
|
||||
Box,
|
||||
useTheme,
|
||||
Card,
|
||||
CardContent,
|
||||
CardActions,
|
||||
} from "@mui/material"
|
||||
import ClearIcon from "@mui/icons-material/Clear"
|
||||
import IconButton from "@mui/material/IconButton"
|
||||
import OpenInFullIcon from "@mui/icons-material/OpenInFull"
|
||||
import Modal from "@mui/material/Modal"
|
||||
import { red } from "@mui/material/colors"
|
||||
|
||||
import { TrialListDetail } from "./TrialList"
|
||||
import { MarkdownRenderer } from "./Note"
|
||||
import { formatDate } from "../dateUtil"
|
||||
|
||||
type TrialType = "worst" | "none"
|
||||
|
||||
const CandidateTrial: FC<{
|
||||
trial: Trial
|
||||
type: TrialType
|
||||
}> = ({ trial, type }) => {
|
||||
const theme = useTheme()
|
||||
const trialWidth = 300
|
||||
const trialHeight = 300
|
||||
const [detailShown, setDetailShown] = useState(false)
|
||||
|
||||
const cardComponentSx = {
|
||||
padding: 0,
|
||||
position: "relative",
|
||||
overflow: "hidden",
|
||||
"::before": {},
|
||||
}
|
||||
if (type !== "none") {
|
||||
cardComponentSx["::before"] = {
|
||||
content: '""',
|
||||
position: "absolute",
|
||||
top: 0,
|
||||
left: 0,
|
||||
width: "100%",
|
||||
height: "100%",
|
||||
backgroundColor: theme.palette.mode === "dark" ? "white" : "black",
|
||||
opacity: 0.2,
|
||||
zIndex: 1,
|
||||
transition: "opacity 0.3s ease-out",
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Card
|
||||
sx={{
|
||||
width: trialWidth,
|
||||
minHeight: trialHeight,
|
||||
margin: theme.spacing(2),
|
||||
padding: 0,
|
||||
}}
|
||||
>
|
||||
<CardActions>
|
||||
<Typography variant="h5">Trial {trial.number}</Typography>
|
||||
<IconButton
|
||||
sx={{
|
||||
marginLeft: "auto",
|
||||
}}
|
||||
onClick={() => setDetailShown(true)}
|
||||
aria-label="show detail"
|
||||
>
|
||||
<OpenInFullIcon />
|
||||
</IconButton>
|
||||
</CardActions>
|
||||
<CardContent aria-label="trial" sx={cardComponentSx}>
|
||||
<Box
|
||||
sx={{
|
||||
padding: theme.spacing(2),
|
||||
}}
|
||||
>
|
||||
<MarkdownRenderer body={trial.note.body} />
|
||||
</Box>
|
||||
|
||||
{type === "worst" ? (
|
||||
<ClearIcon
|
||||
sx={{
|
||||
position: "absolute",
|
||||
width: "100%",
|
||||
height: "100%",
|
||||
top: 0,
|
||||
left: 0,
|
||||
color: red[600],
|
||||
zIndex: 1,
|
||||
opacity: 0.3,
|
||||
filter:
|
||||
theme.palette.mode === "dark"
|
||||
? "brightness(1.1)"
|
||||
: "brightness(1.7)",
|
||||
}}
|
||||
/>
|
||||
) : null}
|
||||
</CardContent>
|
||||
<Modal open={detailShown} onClose={() => setDetailShown(false)}>
|
||||
<Box
|
||||
sx={{
|
||||
position: "absolute",
|
||||
top: 0,
|
||||
left: 0,
|
||||
right: 0,
|
||||
bottom: 0,
|
||||
width: "80%",
|
||||
maxHeight: "90%",
|
||||
margin: "auto",
|
||||
overflow: "hidden",
|
||||
backgroundColor: theme.palette.mode === "dark" ? "black" : "white",
|
||||
borderRadius: theme.spacing(3),
|
||||
}}
|
||||
>
|
||||
<Box
|
||||
sx={{
|
||||
width: "100%",
|
||||
height: "100%",
|
||||
overflow: "auto",
|
||||
}}
|
||||
>
|
||||
<TrialListDetail
|
||||
trial={trial}
|
||||
isBestTrial={() => false}
|
||||
directions={[]}
|
||||
objectiveNames={[]}
|
||||
/>
|
||||
</Box>
|
||||
</Box>
|
||||
</Modal>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
const ChoiceTrials: FC<{ choice: PreferenceHistory; trials: Trial[] }> = ({
|
||||
choice,
|
||||
trials,
|
||||
}) => {
|
||||
const theme = useTheme()
|
||||
const worst_trials = new Set([choice.clicked])
|
||||
|
||||
return (
|
||||
<Box
|
||||
sx={{
|
||||
marginBottom: theme.spacing(4),
|
||||
}}
|
||||
>
|
||||
<Typography
|
||||
variant="h6"
|
||||
sx={{
|
||||
fontWeight: theme.typography.fontWeightLight,
|
||||
}}
|
||||
>
|
||||
{formatDate(choice.timestamp)}
|
||||
</Typography>
|
||||
<Box
|
||||
sx={{
|
||||
display: "flex",
|
||||
flexDirection: "row",
|
||||
flexWrap: "wrap",
|
||||
}}
|
||||
>
|
||||
{choice.candidates.map((trial_num, index) => (
|
||||
<CandidateTrial
|
||||
key={index}
|
||||
trial={trials[trial_num]}
|
||||
type={worst_trials.has(trial_num) ? "worst" : "none"}
|
||||
/>
|
||||
))}
|
||||
</Box>
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
|
||||
export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({
|
||||
studyDetail,
|
||||
}) => {
|
||||
if (
|
||||
studyDetail === null ||
|
||||
!studyDetail.is_preferential ||
|
||||
studyDetail.preference_history === undefined
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const theme = useTheme()
|
||||
const preference_histories = [...studyDetail.preference_history]
|
||||
|
||||
if (preference_histories.length === 0) {
|
||||
return (
|
||||
<Typography
|
||||
variant="h5"
|
||||
sx={{
|
||||
margin: theme.spacing(4),
|
||||
fontWeight: theme.typography.fontWeightBold,
|
||||
}}
|
||||
>
|
||||
No feedback history
|
||||
</Typography>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<Box
|
||||
padding={theme.spacing(2)}
|
||||
sx={{ display: "flex", flexDirection: "column" }}
|
||||
>
|
||||
{preference_histories.reverse().map((choice) => (
|
||||
<ChoiceTrials
|
||||
key={choice.id}
|
||||
choice={choice}
|
||||
trials={studyDetail.trials}
|
||||
/>
|
||||
))}
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
@@ -21,9 +21,9 @@ import { MarkdownRenderer } from "./Note"
|
||||
|
||||
const PreferentialTrial: FC<{
|
||||
trial?: Trial
|
||||
studyDetail: StudyDetail
|
||||
candidates: number[]
|
||||
hideTrial: () => void
|
||||
}> = ({ trial, studyDetail, hideTrial }) => {
|
||||
}> = ({ trial, candidates, hideTrial }) => {
|
||||
const theme = useTheme()
|
||||
const action = actionCreator()
|
||||
const trialWidth = 500
|
||||
@@ -80,10 +80,7 @@ const PreferentialTrial: FC<{
|
||||
aria-label="trial-button"
|
||||
onClick={() => {
|
||||
hideTrial()
|
||||
const best_trials = studyDetail.best_trials
|
||||
.map((t) => t.number)
|
||||
.filter((t) => t !== trial.number)
|
||||
action.updatePreference(trial.study_id, best_trials, [trial.number])
|
||||
action.updatePreference(trial.study_id, candidates, trial.number)
|
||||
}}
|
||||
sx={{
|
||||
padding: 0,
|
||||
@@ -243,7 +240,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({
|
||||
<PreferentialTrial
|
||||
key={index}
|
||||
trial={studyDetail.best_trials.find((trial) => trial.number === t)}
|
||||
studyDetail={studyDetail}
|
||||
candidates={displayTrials.numbers.filter((n) => n !== -1)}
|
||||
hideTrial={() => {
|
||||
hideTrial(t)
|
||||
}}
|
||||
|
||||
@@ -30,6 +30,7 @@ import { GraphEdf } from "./GraphEdf"
|
||||
import { TrialList } from "./TrialList"
|
||||
import { StudyHistory } from "./StudyHistory"
|
||||
import { PreferentialTrials } from "./PreferentialTrials"
|
||||
import { PreferenceHistory } from "./PreferenceHistory"
|
||||
import { PreferentialAnalytics } from "./PreferentialAnalytics"
|
||||
|
||||
interface ParamTypes {
|
||||
@@ -175,6 +176,8 @@ export const StudyDetail: FC<{
|
||||
/>
|
||||
</Box>
|
||||
)
|
||||
} else if (page == "preferenceHistory") {
|
||||
content = <PreferenceHistory studyDetail={studyDetail} />
|
||||
}
|
||||
|
||||
const toolbar = (
|
||||
|
||||
@@ -15,6 +15,7 @@ import { GraphIntermediateValues } from "./GraphIntermediateValues"
|
||||
import Grid2 from "@mui/material/Unstable_Grid2"
|
||||
import { DataGrid, DataGridColumn } from "./DataGrid"
|
||||
import { GraphHyperparameterImportance } from "./GraphHyperparameterImportances"
|
||||
import { UserDefinedPlot } from "./UserDefinedPlot"
|
||||
import { BestTrialsCard } from "./BestTrialsCard"
|
||||
import {
|
||||
useStudyDetailValue,
|
||||
@@ -124,6 +125,16 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => {
|
||||
<Grid2 xs={6}>
|
||||
<GraphTimeline study={studyDetail} />
|
||||
</Grid2>
|
||||
{studyDetail !== null &&
|
||||
studyDetail.plotly_graph_objects.map((go) => (
|
||||
<Grid2 xs={6} key={go.id}>
|
||||
<Card>
|
||||
<CardContent>
|
||||
<UserDefinedPlot graphObject={go} />
|
||||
</CardContent>
|
||||
</Card>
|
||||
</Grid2>
|
||||
))}
|
||||
<Grid2 xs={6} spacing={2}>
|
||||
<BestTrialsCard studyDetail={studyDetail} />
|
||||
</Grid2>
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import * as plotly from "plotly.js-dist-min"
|
||||
import React, { FC, useEffect } from "react"
|
||||
import { Box } from "@mui/material"
|
||||
|
||||
export const UserDefinedPlot: FC<{
|
||||
graphObject: PlotlyGraphObject
|
||||
}> = ({ graphObject }) => {
|
||||
const plotDomId = `user-defined-plot:${graphObject.id}`
|
||||
|
||||
useEffect(() => {
|
||||
try {
|
||||
const parsed = JSON.parse(graphObject.graph_object)
|
||||
plotly.react(plotDomId, parsed.data, parsed.layout)
|
||||
} catch (e) {
|
||||
// Avoid to crash the whole page when given invalid grpah objects.
|
||||
console.error(e)
|
||||
}
|
||||
}, [graphObject])
|
||||
|
||||
return <Box id={plotDomId} sx={{ height: "450px" }} />
|
||||
}
|
||||
Vendored
+17
@@ -12,6 +12,7 @@ type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan"
|
||||
type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting"
|
||||
type TrialStateFinished = "Complete" | "Fail" | "Pruned"
|
||||
type StudyDirection = "maximize" | "minimize" | "not_set"
|
||||
type PreferenceFeedbackMode = "ChooseWorst"
|
||||
|
||||
type FloatDistribution = {
|
||||
type: "FloatDistribution"
|
||||
@@ -181,6 +182,11 @@ type FormWidgets =
|
||||
widgets: UserAttrFormWidget[]
|
||||
}
|
||||
|
||||
type PlotlyGraphObject = {
|
||||
id: string
|
||||
graph_object: string
|
||||
}
|
||||
|
||||
type StudyDetail = {
|
||||
id: number
|
||||
name: string
|
||||
@@ -197,6 +203,8 @@ type StudyDetail = {
|
||||
is_preferential: boolean
|
||||
objective_names?: string[]
|
||||
form_widgets?: FormWidgets
|
||||
preference_history?: PreferenceHistory[]
|
||||
plotly_graph_objects: PlotlyGraphObject[]
|
||||
}
|
||||
|
||||
type StudyDetails = {
|
||||
@@ -206,3 +214,12 @@ type StudyDetails = {
|
||||
type StudyParamImportance = {
|
||||
[study_id: string]: ParamImportance[][]
|
||||
}
|
||||
|
||||
type PreferenceHistory = {
|
||||
id: string
|
||||
preference_id: string
|
||||
candidates: number[]
|
||||
clicked: number
|
||||
feedback_mode: PreferenceFeedbackMode
|
||||
timestamp: Date
|
||||
}
|
||||
|
||||
@@ -43,6 +43,7 @@ docs = [
|
||||
|
||||
test = [
|
||||
"coverage",
|
||||
"plotly",
|
||||
"pytest",
|
||||
"moto[s3]",
|
||||
]
|
||||
|
||||
@@ -135,7 +135,13 @@ class APITestCase(TestCase):
|
||||
app,
|
||||
f"/api/studies/{study_id}/preference",
|
||||
"POST",
|
||||
body=json.dumps({"best_trials": [0, 2], "worst_trials": [1]}),
|
||||
body=json.dumps(
|
||||
{
|
||||
"mode": "ChooseWorst",
|
||||
"candidates": [0, 1, 2],
|
||||
"clicked": 1,
|
||||
}
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
self.assertEqual(status, 204)
|
||||
@@ -150,6 +156,30 @@ class APITestCase(TestCase):
|
||||
assert better.number == 2
|
||||
assert worse.number == 1
|
||||
|
||||
def test_report_preference_when_typo_mode(self) -> None:
|
||||
storage = optuna.storages.InMemoryStorage()
|
||||
study = create_study(storage=storage, n_generate=3)
|
||||
for _ in range(3):
|
||||
trial = study.ask()
|
||||
study.mark_comparison_ready(trial)
|
||||
|
||||
app = create_app(storage)
|
||||
study_id = study._study._study_id
|
||||
status, _, _ = send_request(
|
||||
app,
|
||||
f"/api/studies/{study_id}/preference",
|
||||
"POST",
|
||||
body=json.dumps(
|
||||
{
|
||||
"mode": "ChoseWorst",
|
||||
"candidates": [0, 1, 2],
|
||||
"clicked": 1,
|
||||
}
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
self.assertEqual(status, 400)
|
||||
|
||||
def test_skip_trial(self) -> None:
|
||||
storage = optuna.storages.InMemoryStorage()
|
||||
study = create_study(n_generate=4, storage=storage)
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import optuna
|
||||
from optuna_dashboard import _custom_plot_data as custom_plot_data
|
||||
from optuna_dashboard import save_plotly_graph_object
|
||||
import pytest
|
||||
|
||||
|
||||
def get_dummy_study() -> optuna.Study:
|
||||
def objective(trial: optuna.Trial) -> float:
|
||||
x = trial.suggest_float("x", -100, 100)
|
||||
y = trial.suggest_categorical("y", [-1, 0, 1])
|
||||
return x**2 + y
|
||||
|
||||
study = optuna.create_study()
|
||||
optuna.logging.set_verbosity(optuna.logging.ERROR)
|
||||
study.optimize(objective, n_trials=100)
|
||||
return study
|
||||
|
||||
|
||||
def test_save_plotly_graph_object() -> None:
|
||||
# Save history plot
|
||||
dummy_study = get_dummy_study()
|
||||
plot_data = optuna.visualization.plot_optimization_history(dummy_study)
|
||||
graph_object_id = save_plotly_graph_object(dummy_study, plot_data)
|
||||
|
||||
study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id)
|
||||
plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs)
|
||||
assert len(plot_data_dict) == 1
|
||||
assert plot_data_dict[graph_object_id] == plot_data.to_json()
|
||||
|
||||
# Save parallel coordinate plot
|
||||
plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study)
|
||||
graph_object_id = save_plotly_graph_object(dummy_study, plot_data)
|
||||
|
||||
study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id)
|
||||
plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs)
|
||||
assert len(plot_data_dict) == 2
|
||||
assert plot_data_dict[graph_object_id] == plot_data.to_json()
|
||||
|
||||
|
||||
def test_update_plotly_graph_object() -> None:
|
||||
# Save history plot
|
||||
dummy_study = get_dummy_study()
|
||||
plot_data = optuna.visualization.plot_optimization_history(dummy_study)
|
||||
graph_object_id = save_plotly_graph_object(dummy_study, plot_data)
|
||||
|
||||
study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id)
|
||||
plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs)
|
||||
assert len(plot_data_dict) == 1
|
||||
assert plot_data_dict[graph_object_id] == plot_data.to_json()
|
||||
|
||||
# Save parallel coordinate plot
|
||||
plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study)
|
||||
graph_object_id = save_plotly_graph_object(
|
||||
dummy_study, plot_data, graph_object_id=graph_object_id
|
||||
)
|
||||
|
||||
study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id)
|
||||
plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs)
|
||||
assert len(plot_data_dict) == 1
|
||||
assert plot_data_dict[graph_object_id] == plot_data.to_json()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"0",
|
||||
"a",
|
||||
"a1-:_.",
|
||||
],
|
||||
)
|
||||
def test_is_valid_graph_object_id(name: str) -> None:
|
||||
assert custom_plot_data.is_valid_graph_object_id(name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"a,",
|
||||
"a b",
|
||||
"aあいうえお",
|
||||
],
|
||||
)
|
||||
def test_is_invalid_graph_object_id(name: str) -> None:
|
||||
assert not custom_plot_data.is_valid_graph_object_id(name)
|
||||
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable
|
||||
|
||||
from optuna_dashboard._preferential_history import NewHistory
|
||||
from optuna_dashboard._preferential_history import report_history
|
||||
from optuna_dashboard._serializer import serialize_preference_history
|
||||
from optuna_dashboard.preferential import create_study
|
||||
from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE
|
||||
|
||||
from .storage_supplier import parametrize_storages
|
||||
from .storage_supplier import StorageSupplier
|
||||
|
||||
|
||||
@parametrize_storages
|
||||
def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None:
|
||||
with storage_supplier() as storage:
|
||||
study = create_study(storage=storage, n_generate=5)
|
||||
for _ in range(5):
|
||||
trial = study.ask()
|
||||
trial.suggest_float("x", 0, 1)
|
||||
study.mark_comparison_ready(trial)
|
||||
|
||||
study_id = study._study._study_id
|
||||
|
||||
report_history(
|
||||
study_id=study_id,
|
||||
storage=storage,
|
||||
input_data=NewHistory(
|
||||
mode="ChooseWorst",
|
||||
candidates=[0, 1, 2],
|
||||
clicked=1,
|
||||
),
|
||||
)
|
||||
report_history(
|
||||
study_id=study_id,
|
||||
storage=storage,
|
||||
input_data=NewHistory(
|
||||
mode="ChooseWorst",
|
||||
candidates=[0, 2, 3, 4],
|
||||
clicked=0,
|
||||
),
|
||||
)
|
||||
history = serialize_preference_history(storage.get_study_system_attrs(study_id))
|
||||
sys_attrs = storage.get_study_system_attrs(study_id)
|
||||
assert len(history) == 2
|
||||
assert history[0]["candidates"] == [0, 1, 2]
|
||||
assert history[0]["clicked"] == 1
|
||||
preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]]
|
||||
assert len(preferences) == 2
|
||||
for i, (best, worst) in enumerate([(0, 1), (2, 1)]):
|
||||
assert len(preferences[i]) == 2
|
||||
assert preferences[i][0] == best
|
||||
assert preferences[i][1] == worst
|
||||
assert history[1]["candidates"] == [0, 2, 3, 4]
|
||||
assert history[1]["clicked"] == 0
|
||||
preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]]
|
||||
assert len(preferences) == 3
|
||||
for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]):
|
||||
assert len(preferences[i]) == 2
|
||||
assert preferences[i][0] == best
|
||||
assert preferences[i][1] == worst
|
||||
@@ -29,7 +29,7 @@ def test_get_study_detail_is_preferential() -> None:
|
||||
assert len(study_summaries) == 1
|
||||
|
||||
study_summary = study_summaries[0]
|
||||
study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False)
|
||||
study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {})
|
||||
assert study_detail["is_preferential"]
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ def test_get_study_detail_is_not_preferential() -> None:
|
||||
assert len(study_summaries) == 1
|
||||
|
||||
study_summary = study_summaries[0]
|
||||
study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False)
|
||||
study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {})
|
||||
assert not study_detail["is_preferential"]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user