diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 8183f48d..d3170fad 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -336,27 +336,26 @@ def create_app(storage: BaseStorage, debug: bool = False) -> Bottle: @app.get("/api/studies//param_importances") @json_api_view def get_param_importances(study_id: int) -> BottleViewReturn: - # TODO(chenghuzi): add support for selecting params via query parameters. - objective_id = int(request.params.get("objective_id", 0)) try: n_directions = len(storage.get_study_directions(study_id)) except KeyError: response.status = 404 # Study is not found return {"reason": f"study_id={study_id} is not found"} - if objective_id >= n_directions: - response.status = 400 # Bad request - return {"reason": f"study_id={study_id} has only {n_directions} direction(s)."} trials = get_trials(storage, study_id) try: - return get_param_importance_from_trials_cache(storage, study_id, objective_id, trials) + importances = [ + get_param_importance_from_trials_cache(storage, study_id, objective_id, trials) + for objective_id in range(n_directions) + ] + return {"param_importances": importances} except ValueError as e: response.status = 400 # Bad request return {"reason": str(e)} @app.put("/api/studies//note") @json_api_view - def save_note(study_id: int) -> BottleViewReturn: + def save_study_note(study_id: int) -> BottleViewReturn: req_note_ver = request.json.get("version", None) req_note_body = request.json.get("body", None) if req_note_ver is None or req_note_body is None: @@ -364,15 +363,38 @@ def create_app(storage: BaseStorage, debug: bool = False) -> Bottle: return {"reason": "Invalid request."} system_attrs = storage.get_study_system_attrs(study_id) - if not note.version_is_incremented(system_attrs, req_note_ver): + if not note.version_is_incremented(system_attrs, None, req_note_ver): response.status = 409 # Conflict return { "reason": "The text you are editing has changed. " "Please copy your edits and refresh the page.", - "note": note.get_note_from_system_attrs(system_attrs), + "note": note.get_note_from_system_attrs(system_attrs, None), } - note.save_note(storage, study_id, req_note_ver, req_note_body) + note.save_note(storage, study_id, None, req_note_ver, req_note_body) + response.status = 204 # No content + return {} + + @app.put("/api/studies///note") + @json_api_view + def save_trial_note(study_id: int, trial_id: int) -> BottleViewReturn: + req_note_ver = request.json.get("version", None) + req_note_body = request.json.get("body", None) + if req_note_ver is None or req_note_body is None: + response.status = 400 # Bad request + return {"reason": "Invalid request."} + + # Store note content in study system attrs since it's always updatable. + system_attrs = storage.get_study_system_attrs(study_id=study_id) + if not note.version_is_incremented(system_attrs, trial_id, req_note_ver): + response.status = 409 # Conflict + return { + "reason": "The text you are editing has changed. " + "Please copy your edits and refresh the page.", + "note": note.get_note_from_system_attrs(system_attrs, trial_id), + } + + note.save_note(storage, study_id, trial_id, req_note_ver, req_note_body) response.status = 204 # No content return {} diff --git a/optuna_dashboard/_importance.py b/optuna_dashboard/_importance.py index f6e0e4fa..5b375597 100644 --- a/optuna_dashboard/_importance.py +++ b/optuna_dashboard/_importance.py @@ -25,26 +25,18 @@ except Exception as e: if TYPE_CHECKING: from typing import TypedDict - ImportanceItemType = TypedDict( - "ImportanceItemType", + ImportanceType = TypedDict( + "ImportanceType", { "name": str, "importance": float, "distribution": str, }, ) - ImportanceType = TypedDict( - "ImportanceType", - { - "target_name": str, - "param_importances": list[ImportanceItemType], - }, - ) -target_name = "Objective Value" param_importance_cache_lock = threading.Lock() # { "{study_id}:{objective_id}" : (n_completed_trials, importance) } -param_importance_cache: dict[str, tuple[int, ImportanceType]] = {} +param_importance_cache: dict[str, tuple[int, list[ImportanceType]]] = {} class StudyWrapper(Study): @@ -62,17 +54,15 @@ class StudyWrapper(Study): def get_param_importance_from_trials_cache( storage: BaseStorage, study_id: int, objective_id: int, trials: list[FrozenTrial] -) -> ImportanceType: +) -> list[ImportanceType]: completed_trials = [t for t in trials if t.state == TrialState.COMPLETE] n_completed_trials = len(completed_trials) if n_completed_trials == 0: - return {"target_name": target_name, "param_importances": []} + return [] cache_key = f"{study_id}:{objective_id}" with param_importance_cache_lock: - cache_n_trial, cache_importance = param_importance_cache.get( - cache_key, (0, {"target_name": target_name, "param_importances": []}) - ) + cache_n_trial, cache_importance = param_importance_cache.get(cache_key, (0, [])) if n_completed_trials == cache_n_trial: return cache_importance @@ -95,18 +85,15 @@ def get_param_importance_from_trials_cache( def convert_to_importance_type( importance: dict[str, float], trials: list[FrozenTrial] -) -> ImportanceType: - return { - "target_name": target_name, - "param_importances": [ - { - "name": name, - "importance": importance, - "distribution": get_distribution_name(name, trials), - } - for name, importance in importance.items() - ], - } +) -> list[ImportanceType]: + return [ + { + "name": name, + "importance": importance, + "distribution": get_distribution_name(name, trials), + } + for name, importance in importance.items() + ] def get_distribution_name(param_name: str, trials: list[FrozenTrial]) -> str: diff --git a/optuna_dashboard/_note.py b/optuna_dashboard/_note.py index 00710276..4e872e1e 100644 --- a/optuna_dashboard/_note.py +++ b/optuna_dashboard/_note.py @@ -8,6 +8,7 @@ from optuna.storages import BaseStorage if TYPE_CHECKING: + from typing import Optional from typing import TypedDict NoteType = TypedDict( @@ -19,32 +20,50 @@ if TYPE_CHECKING: ) SYSTEM_ATTR_MAX_LENGTH = 2045 -NOTE_VER_KEY = "dashboard:note_ver" -NOTE_STR_KEY_PREFIX = "dashboard:note_str:" -def get_note_from_system_attrs(system_attrs: dict[str, Any]) -> NoteType: - if NOTE_VER_KEY not in system_attrs: +def note_ver_key(trial_id: Optional[int]) -> str: + prefix = "dashboard:note_ver" + if trial_id is None: + return prefix + return f"dashboard:{trial_id}:note_ver" + + +def note_str_key_prefix(trial_id: Optional[int]) -> str: + prefix = "dashboard:note_str:" + if trial_id is None: + return prefix + return f"dashboard:{trial_id}:note_str:" + + +def get_note_from_system_attrs(system_attrs: dict[str, Any], trial_id: Optional[int]) -> NoteType: + if note_ver_key(trial_id) not in system_attrs: return { "version": 0, "body": "", } - note_ver = int(system_attrs[NOTE_VER_KEY]) + note_ver = int(system_attrs[note_ver_key(trial_id)]) note_attrs: dict[str, str] = { - key: value for key, value in system_attrs.items() if key.startswith(NOTE_STR_KEY_PREFIX) + key: value + for key, value in system_attrs.items() + if key.startswith(note_str_key_prefix(trial_id)) } - return {"version": note_ver, "body": concat_body(note_attrs)} + return {"version": note_ver, "body": concat_body(note_attrs, trial_id)} -def version_is_incremented(system_attrs: dict[str, Any], req_note_ver: int) -> bool: - db_note_ver = system_attrs.get(NOTE_VER_KEY, 0) +def version_is_incremented( + system_attrs: dict[str, Any], trial_id: Optional[int], req_note_ver: int +) -> bool: + db_note_ver = system_attrs.get(note_ver_key(trial_id), 0) return req_note_ver == db_note_ver + 1 -def save_note(storage: BaseStorage, study_id: int, ver: int, body: str) -> None: - storage.set_study_system_attr(study_id, NOTE_VER_KEY, ver) +def save_note( + storage: BaseStorage, study_id: int, trial_id: Optional[int], ver: int, body: str +) -> None: + storage.set_study_system_attr(study_id, note_ver_key(trial_id), ver) - attrs = split_body(body) + attrs = split_body(body, trial_id) for k, v in attrs.items(): storage.set_study_system_attr(study_id, k, v) @@ -52,22 +71,24 @@ def save_note(storage: BaseStorage, study_id: int, ver: int, body: str) -> None: all_note_attrs: dict[str, str] = { key: value for key, value in storage.get_study_system_attrs(study_id).items() - if key.startswith(NOTE_STR_KEY_PREFIX) + if key.startswith(note_str_key_prefix(trial_id)) } if len(all_note_attrs) > len(attrs): for i in range(len(attrs), len(all_note_attrs)): - storage.set_study_system_attr(study_id, f"{NOTE_STR_KEY_PREFIX}{i}", "") + storage.set_study_system_attr(study_id, f"{note_str_key_prefix(trial_id)}{i}", "") -def split_body(note_str: str) -> dict[str, str]: +def split_body(note_str: str, trial_id: Optional[int]) -> dict[str, str]: note_len = len(note_str) attrs = {} for i in range(math.ceil(note_len / SYSTEM_ATTR_MAX_LENGTH)): start = i * SYSTEM_ATTR_MAX_LENGTH end = min((i + 1) * SYSTEM_ATTR_MAX_LENGTH, note_len) - attrs[f"{NOTE_STR_KEY_PREFIX}{i}"] = note_str[start:end] + attrs[f"{note_str_key_prefix(trial_id)}{i}"] = note_str[start:end] return attrs -def concat_body(note_attrs: dict[str, str]) -> str: - return "".join(note_attrs[f"{NOTE_STR_KEY_PREFIX}{i}"] for i in range(len(note_attrs))) +def concat_body(note_attrs: dict[str, str], trial_id: Optional[int]) -> str: + return "".join( + note_attrs[f"{note_str_key_prefix(trial_id)}{i}"] for i in range(len(note_attrs)) + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 4e426849..764083b3 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -84,22 +84,27 @@ def serialize_study_detail( "name": summary.study_name, "directions": [d.name.lower() for d in summary.directions], } + system_attrs = getattr(summary, "system_attrs", {}) if summary.datetime_start is not None: serialized["datetime_start"] = summary.datetime_start.isoformat() - serialized["trials"] = [serialize_frozen_trial(summary._study_id, trial) for trial in trials] + serialized["trials"] = [ + serialize_frozen_trial(summary._study_id, trial, system_attrs) for trial in trials + ] serialized["best_trials"] = [ - serialize_frozen_trial(summary._study_id, trial) for trial in best_trials + serialize_frozen_trial(summary._study_id, trial, system_attrs) for trial in best_trials ] serialized["intersection_search_space"] = serialize_search_space(intersection) serialized["union_search_space"] = serialize_search_space(union) serialized["union_user_attrs"] = [{"key": a[0], "sortable": a[1]} for a in union_user_attrs] serialized["has_intermediate_values"] = has_intermediate_values - serialized["note"] = note.get_note_from_system_attrs(getattr(summary, "system_attrs", {})) + serialized["note"] = note.get_note_from_system_attrs(system_attrs, None) return serialized -def serialize_frozen_trial(study_id: int, trial: FrozenTrial) -> dict[str, Any]: +def serialize_frozen_trial( + study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any] +) -> dict[str, Any]: serialized = { "trial_id": trial._trial_id, "study_id": study_id, @@ -108,6 +113,7 @@ def serialize_frozen_trial(study_id: int, trial: FrozenTrial) -> dict[str, Any]: "params": [{"name": name, "value": str(value)} for name, value in trial.params.items()], "user_attrs": serialize_attrs(trial.user_attrs), "system_attrs": serialize_attrs(getattr(trial, "_system_attrs", {})), + "note": note.get_note_from_system_attrs(study_system_attrs, trial._trial_id), } serialized_intermediate_values: list[IntermediateValue] = [] diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index da87dc3f..6d5aa0d8 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -3,14 +3,17 @@ import { useSnackbar } from "notistack" import { getStudyDetailAPI, getStudySummariesAPI, + getParamImportances, createNewStudyAPI, deleteStudyAPI, - saveNoteAPI, + saveStudyNoteAPI, + saveTrialNoteAPI, } from "./apiClient" import { graphVisibilityState, studyDetailsState, studySummariesState, + paramImportanceState, } from "./state" const localStorageGraphVisibility = "graphVisibility" @@ -23,6 +26,8 @@ export const actionCreator = () => { useRecoilState(studyDetailsState) const [graphVisibility, setGraphVisibility] = useRecoilState(graphVisibilityState) + const [paramImportance, setParamImportance] = + useRecoilState(paramImportanceState) const setStudyDetailState = (studyId: number, study: StudyDetail) => { const newVal = Object.assign({}, studyDetails) @@ -30,6 +35,28 @@ export const actionCreator = () => { setStudyDetails(newVal) } + const setTrialNote = (studyId: number, index: number, note: Note) => { + const newTrial: Trial = Object.assign( + {}, + studyDetails[studyId].trials[index] + ) + newTrial.note = note + const newTrials: Trial[] = [...studyDetails[studyId].trials] + newTrials[index] = newTrial + const newStudy: StudyDetail = Object.assign({}, studyDetails[studyId]) + newStudy.trials = newTrials + setStudyDetailState(studyId, newStudy) + } + + const setStudyParamImportanceState = ( + studyId: number, + importance: ParamImportance[][] + ) => { + const newVal = Object.assign({}, paramImportance) + newVal[studyId] = importance + setParamImportance(newVal) + } + const updateStudySummaries = (successMsg?: string) => { getStudySummariesAPI() .then((studySummaries: StudySummary[]) => { @@ -77,6 +104,22 @@ export const actionCreator = () => { }) } + const updateParamImportance = (studyId: number) => { + getParamImportances(studyId) + .then((importance) => { + setStudyParamImportanceState(studyId, importance) + }) + .catch((err) => { + const reason = err.response?.data.reason + enqueueSnackbar( + `Failed to load hyperparameter importance (reason=${reason})`, + { + variant: "error", + } + ) + }) + } + const createNewStudy = (studyName: string, directions: StudyDirection[]) => { createNewStudyAPI(studyName, directions) .then((study_summary) => { @@ -128,8 +171,8 @@ export const actionCreator = () => { localStorage.setItem(localStorageGraphVisibility, JSON.stringify(value)) } - const saveNote = (studyId: number, note: Note): Promise => { - return saveNoteAPI(studyId, note) + const saveStudyNote = (studyId: number, note: Note): Promise => { + return saveStudyNoteAPI(studyId, note) .then(() => { const newStudy = Object.assign({}, studyDetails[studyId]) newStudy.note = note @@ -154,14 +197,64 @@ export const actionCreator = () => { }) } + const saveTrialNote = ( + studyId: number, + trialId: number, + note: Note + ): Promise => { + return saveTrialNoteAPI(studyId, trialId, note) + .then(() => { + const index = studyDetails[studyId].trials.findIndex( + (t) => t.trial_id === trialId + ) + if (index === -1) { + enqueueSnackbar(`Unexpected error happens. Please reload the page.`, { + variant: "error", + }) + return + } + setTrialNote(studyId, index, note) + enqueueSnackbar(`Success to save the note`, { + variant: "success", + }) + }) + .catch((err) => { + console.dir(err) + if (err.response.status === 409) { + const index = studyDetails[studyId].trials.findIndex( + (t) => t.trial_id === trialId + ) + if (index === -1) { + enqueueSnackbar( + `Unexpected error happens. Please reload the page.`, + { + variant: "error", + } + ) + return + } + setTrialNote(studyId, index, note) + } + const reason = err.response?.data.reason + if (reason !== undefined) { + enqueueSnackbar(`Failed: ${reason}`, { + variant: "error", + }) + } + throw err + }) + } + return { updateStudyDetail, updateStudySummaries, + updateParamImportance, createNewStudy, deleteStudy, getGraphVisibility, saveGraphVisibility, - saveNote, + saveStudyNote, + saveTrialNote, } } diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index d7812a56..e1718b9c 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -14,6 +14,7 @@ interface TrialResponse { params: TrialParam[] user_attrs: Attribute[] system_attrs: Attribute[] + note: Note } const convertTrialResponse = (res: TrialResponse): Trial => { @@ -33,6 +34,7 @@ const convertTrialResponse = (res: TrialResponse): Trial => { params: res.params, user_attrs: res.user_attrs, system_attrs: res.system_attrs, + note: res.note, } } @@ -46,10 +48,7 @@ interface StudyDetailResponse { union_search_space: SearchSpace[] union_user_attrs: AttributeSpec[] has_intermediate_values: boolean - note: { - version: number - body: string - } + note: Note } export const getStudyDetailAPI = ( @@ -183,7 +182,7 @@ export const deleteStudyAPI = (studyId: number) => { }) } -export const saveNoteAPI = ( +export const saveStudyNoteAPI = ( studyId: number, note: { version: number; body: string } ): Promise => { @@ -194,25 +193,28 @@ export const saveNoteAPI = ( }) } +export const saveTrialNoteAPI = ( + studyId: number, + trialId: number, + note: { version: number; body: string } +): Promise => { + return axiosInstance + .put(`/api/studies/${studyId}/${trialId}/note`, note) + .then((res) => { + return + }) +} + interface ParamImportancesResponse { - target_name: string - param_importances: ParamImportance[] + param_importances: ParamImportance[][] } export const getParamImportances = ( - studyId: number, - objectiveId = 0 -): Promise => { + studyId: number +): Promise => { return axiosInstance - .get( - `/api/studies/${studyId}/param_importances`, - { - params: { - objective_id: objectiveId, - }, - } - ) + .get(`/api/studies/${studyId}/param_importances`) .then((res) => { - return res.data + return res.data.param_importances }) } diff --git a/optuna_dashboard/ts/components/App.tsx b/optuna_dashboard/ts/components/App.tsx index 35bacff2..302278d9 100644 --- a/optuna_dashboard/ts/components/App.tsx +++ b/optuna_dashboard/ts/components/App.tsx @@ -76,7 +76,16 @@ export const App: FC = () => { children={ + } + /> + } /> diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 44154bae..713f978d 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -14,9 +14,10 @@ import ListItem from "@mui/material/ListItem" import ListItemButton from "@mui/material/ListItemButton" import ListItemIcon from "@mui/material/ListItemIcon" import ListItemText from "@mui/material/ListItemText" -import { reloadIntervalState } from "../state" +import { drawerOpenState, reloadIntervalState } from "../state" import { Link } from "react-router-dom" import AutoGraphIcon from "@mui/icons-material/AutoGraph" +import ViewListIcon from "@mui/icons-material/ViewList" import SyncIcon from "@mui/icons-material/Sync" import SyncDisabledIcon from "@mui/icons-material/SyncDisabled" import Brightness4Icon from "@mui/icons-material/Brightness4" @@ -104,12 +105,12 @@ const Drawer = styled(MuiDrawer, { export const AppDrawer: FC<{ studyId?: number toggleColorMode: () => void - page?: "history" | "analytics" | "trials" | "note" + page?: PageId toolbar: React.ReactNode children?: React.ReactNode }> = ({ studyId, toggleColorMode, page, toolbar, children }) => { const theme = useTheme() - const [open, setOpen] = React.useState(false) + const [open, setOpen] = useRecoilState(drawerOpenState) const [reloadInterval, updateReloadInterval] = useRecoilState(reloadIntervalState) @@ -199,17 +200,30 @@ export const AppDrawer: FC<{ - + + + + + + + + + - + @@ -301,7 +315,10 @@ export const AppDrawer: FC<{ - + diff --git a/optuna_dashboard/ts/components/GraphHistory.tsx b/optuna_dashboard/ts/components/GraphHistory.tsx index eab76447..da431167 100644 --- a/optuna_dashboard/ts/components/GraphHistory.tsx +++ b/optuna_dashboard/ts/components/GraphHistory.tsx @@ -127,6 +127,7 @@ export const GraphHistory: FC<{ control={ } diff --git a/optuna_dashboard/ts/components/GraphHyperparameterImportances.tsx b/optuna_dashboard/ts/components/GraphHyperparameterImportances.tsx index fa7451e3..373add6c 100644 --- a/optuna_dashboard/ts/components/GraphHyperparameterImportances.tsx +++ b/optuna_dashboard/ts/components/GraphHyperparameterImportances.tsx @@ -10,51 +10,141 @@ import { SelectChangeEvent, useTheme, Box, + Card, + CardContent, } from "@mui/material" +import Grid2 from "@mui/material/Unstable_Grid2" -import { getParamImportances } from "../apiClient" import { plotlyDarkTemplate } from "./PlotlyDarkMode" -import { useSnackbar } from "notistack" +import { actionCreator } from "../action" +import { useParamImportanceValue, useStudyDirections } from "../state" const plotDomId = "graph-hyperparameter-importances" +const getPlotDomId = (objectiveId: number) => `graph-importance-${objectiveId}` + +export const GraphHyperparameterImportanceBeta: FC<{ + studyId: number + study: StudyDetail | null + graphHeight: string +}> = ({ studyId, study = null, graphHeight }) => { + const theme = useTheme() + const action = actionCreator() + const importances = useParamImportanceValue(studyId) + const numCompletedTrials = + study?.trials.filter((t) => t.state === "Complete").length || 0 + const nObjectives = useStudyDirections(studyId)?.length + + useEffect(() => { + action.updateParamImportance(studyId) + }, [numCompletedTrials]) + + useEffect(() => { + if (importances !== null && nObjectives === importances.length) { + plotParamImportancesBeta(importances, theme.palette.mode) + } + }, [nObjectives, importances, theme.palette.mode]) + + return ( + <> + {Array.from({ length: nObjectives || 1 }, (_, i) => { + let title = `Importance for the Objective Value` + if (nObjectives != null && nObjectives > 1) { + title = `Importance for the Objective ${i}` + } + return ( + + + + + {title} + + + + + + ) + })} + + ) +} + +const plotParamImportancesBeta = ( + importances: ParamImportance[][], + mode: string +) => { + const layout: Partial = { + xaxis: { + title: "Hyperparameter Importance", + }, + yaxis: { + title: "Hyperparameter", + automargin: true, + }, + margin: { + l: 50, + t: 0, + r: 50, + b: 50, + }, + showlegend: false, + template: mode === "dark" ? plotlyDarkTemplate : {}, + } + + importances.forEach((importance, objectiveId) => { + if (document.getElementById(getPlotDomId(objectiveId)) === null) { + return + } + + const reversed = [...importance].reverse() + const importance_values = reversed.map((p) => p.importance) + const param_names = reversed.map((p) => p.name) + const param_hover_templates = reversed.map( + (p) => `${p.name} (${p.distribution}): ${p.importance} ` + ) + const plotData: Partial[] = [ + { + type: "bar", + orientation: "h", + x: importance_values, + y: param_names, + text: importance_values.map((v) => String(v.toFixed(2))), + textposition: "outside", + hovertemplate: param_hover_templates, + marker: { + color: "rgb(66,146,198)", + }, + }, + ] + plotly.react(getPlotDomId(objectiveId), plotData, layout) + }) +} export const GraphHyperparameterImportances: FC<{ study: StudyDetail | null studyId: number }> = ({ study = null, studyId }) => { const theme = useTheme() + const action = actionCreator() + const importances = useParamImportanceValue(studyId) const [objectiveId, setObjectiveId] = useState(0) const numCompletedTrials = study?.trials.filter((t) => t.state === "Complete").length || 0 - const [importances, setImportances] = useState(null) - const { enqueueSnackbar } = useSnackbar() const handleObjectiveChange = (event: SelectChangeEvent) => { setObjectiveId(event.target.value as number) } useEffect(() => { - if (numCompletedTrials > 0) { - getParamImportances(studyId, objectiveId) - .then((p) => { - setImportances(p) - }) - .catch((err) => { - const reason = err.response?.data.reason - enqueueSnackbar( - `Failed to load hyperparameter importance (reason=${reason})`, - { - variant: "error", - } - ) - }) - } - }, [numCompletedTrials, objectiveId, theme.palette.mode]) + action.updateParamImportance(studyId) + }, [numCompletedTrials]) useEffect(() => { - if (importances !== null) { - plotParamImportances(importances, theme.palette.mode) + if (importances !== null && importances.length > objectiveId) { + plotParamImportances(importances[objectiveId], theme.palette.mode) } - }, [importances, theme.palette.mode]) + }, [importances, objectiveId, theme.palette.mode]) return ( @@ -88,25 +178,20 @@ export const GraphHyperparameterImportances: FC<{ ) } -const plotParamImportances = ( - paramsImportanceData: ParamImportances, - mode: string -) => { +const plotParamImportances = (importance: ParamImportance[], mode: string) => { if (document.getElementById(plotDomId) === null) { return } - const param_importances = [ - ...paramsImportanceData.param_importances, - ].reverse() - const importance_values = param_importances.map((p) => p.importance) - const param_names = param_importances.map((p) => p.name) - const param_hover_templates = param_importances.map( + const reversed = [...importance].reverse() + const importance_values = reversed.map((p) => p.importance) + const param_names = reversed.map((p) => p.name) + const param_hover_templates = reversed.map( (p) => `${p.name} (${p.distribution}): ${p.importance} ` ) const layout: Partial = { xaxis: { - title: `Importance for ${paramsImportanceData.target_name}`, + title: `Importance for the Objective Value`, }, yaxis: { title: "Hyperparameter", diff --git a/optuna_dashboard/ts/components/GraphSlice.tsx b/optuna_dashboard/ts/components/GraphSlice.tsx index 609e8aab..2a101237 100644 --- a/optuna_dashboard/ts/components/GraphSlice.tsx +++ b/optuna_dashboard/ts/components/GraphSlice.tsx @@ -61,7 +61,6 @@ export const GraphSlice: FC<{ } const handleLogYScaleChange = (e: ChangeEvent) => { - e.preventDefault() setLogYScale(!logYScale) } diff --git a/optuna_dashboard/ts/components/Note.tsx b/optuna_dashboard/ts/components/Note.tsx index 16422395..a8d7ee9b 100644 --- a/optuna_dashboard/ts/components/Note.tsx +++ b/optuna_dashboard/ts/components/Note.tsx @@ -3,22 +3,96 @@ import { Button, Card, CardContent, + CardHeader, + IconButton, + SxProps, TextField, Typography, useTheme, } from "@mui/material" import React, { FC, createRef, useState, useEffect } from "react" +import ReactMarkdown from "react-markdown" +import remarkGfm from "remark-gfm" import LoadingButton from "@mui/lab/LoadingButton" import SaveIcon from "@mui/icons-material/Save" +import EditIcon from "@mui/icons-material/Edit" +import CloseIcon from "@mui/icons-material/Close" +import Divider from "@mui/material/Divider" +import { Theme } from "@mui/material/styles" +import { + CodeComponent, + ReactMarkdownNames, +} from "react-markdown/lib/ast-to-react" +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter" +import { darcula } from "react-syntax-highlighter/dist/esm/styles/prism" import { actionCreator } from "../action" -export const Note: FC<{ +const CodeBlock: CodeComponent | ReactMarkdownNames = ({ + inline, + className, + children, + ...props +}) => { + const match = /language-(\w+)/.exec(className || "") + return !inline && match ? ( + + {String(children).replace(/\n$/, "")} + + ) : ( + + {children} + + ) +} + +export const TrialNote: FC<{ + studyId: number + trialId: number + latestNote: Note + cardSx?: SxProps +}> = ({ studyId, trialId, latestNote, cardSx }) => { + return ( + + ) +} + +export const StudyNote: FC<{ studyId: number latestNote: Note minRows: number -}> = ({ studyId, latestNote, minRows }) => { + cardSx?: SxProps +}> = ({ studyId, latestNote, minRows, cardSx }) => { + return ( + + ) +} + +const NoteBase: FC<{ + studyId: number + trialId?: number + latestNote: Note + minRows: number + cardSx?: SxProps +}> = ({ studyId, trialId, latestNote, minRows, cardSx }) => { const theme = useTheme() + const [renderMarkdown, setRenderMarkdown] = useState(true) const [saving, setSaving] = useState(false) const [edited, setEdited] = useState(false) const [curNote, setCurNote] = useState({ version: 0, body: "" }) @@ -45,10 +119,17 @@ export const Note: FC<{ body: textAreaRef.current ? textAreaRef.current.value : "", } setSaving(true) - action - .saveNote(studyId, newNote) + + let actionResponse: Promise + if (trialId === undefined) { + actionResponse = action.saveStudyNote(studyId, newNote) + } else { + actionResponse = action.saveTrialNote(studyId, trialId, newNote) + } + actionResponse .then(() => { setCurNote(newNote) + setRenderMarkdown(true) window.onbeforeunload = null }) .finally(() => { @@ -65,17 +146,27 @@ export const Note: FC<{ window.onbeforeunload = null } - return ( - - - - Note - + let content + if (renderMarkdown) { + const defaultBody = + "*A markdown editor for taking a memo, related to the study. Click the 'Edit' button in the upper right corner to access the editor.*" + content = ( + + ) + } else { + content = ( + <> + + ) + } + + return ( + + { + setRenderMarkdown(true) + }} + > + + + ) : ( + setRenderMarkdown(false)}> + + + ) + } + sx={{ paddingBottom: 0 }} + /> + + + {content} ) diff --git a/optuna_dashboard/ts/components/StudyDetail.tsx b/optuna_dashboard/ts/components/StudyDetail.tsx index 2058424a..ad06611a 100644 --- a/optuna_dashboard/ts/components/StudyDetail.tsx +++ b/optuna_dashboard/ts/components/StudyDetail.tsx @@ -24,7 +24,7 @@ import { GraphIntermediateValues } from "./GraphIntermediateValues" import { GraphSlice } from "./GraphSlice" import { GraphHistory } from "./GraphHistory" import { GraphParetoFront } from "./GraphParetoFront" -import { Note } from "./Note" +import { StudyNote } from "./Note" import { actionCreator } from "../action" import { graphVisibilityState, @@ -235,7 +235,7 @@ export const StudyDetail: FC<{ {studyDetail !== null ? ( - { - const studyDetails = useRecoilValue(studyDetailsState) - return studyDetails[studyId] || null -} - -const useStudySummaryValue = (studyId: number): StudySummary | null => { - const studySummaries = useRecoilValue(studySummariesState) - return studySummaries.find((s) => s.study_id == studyId) || null -} - export const StudyDetailBeta: FC<{ toggleColorMode: () => void page: PageId @@ -60,20 +50,19 @@ export const StudyDetailBeta: FC<{ const studyDetail = useStudyDetailValue(studyIdNumber) const reloadInterval = useRecoilValue(reloadIntervalState) const studySummary = useStudySummaryValue(studyIdNumber) - const directions = studyDetail?.directions || studySummary?.directions || null + const directions = useStudyDirections(studyIdNumber) + const studyName = useStudyName(studyIdNumber) const userAttrs = studySummary?.user_attrs || [] const title = - studyDetail !== null || studySummary !== null - ? `${studyDetail?.name || studySummary?.study_name} (id=${studyId})` - : `Study #${studyId}` + studyName !== null ? `${studyName} (id=${studyId})` : `Study #${studyId}` useEffect(() => { action.updateStudyDetail(studyIdNumber) }, []) useEffect(() => { - if (reloadInterval < 0 || page === "trials") { + if (reloadInterval < 0 || page === "trialTable" || page === "trialList") { return } const intervalId = setInterval(function () { @@ -92,6 +81,13 @@ export const StudyDetailBeta: FC<{ if (page === "history") { content = ( + {directions !== null && directions.length > 1 ? ( + + + + + + ) : null} ) : null} - {directions !== null && directions.length > 1 ? ( - - - - - - ) : null} - - - - + + + + + {studyDetail !== null && studyDetail.best_trials.length === 1 && ( <> @@ -131,37 +131,89 @@ export const StudyDetailBeta: FC<{ Best Trial {studyDetail.best_trials[0].values} - - {studyDetail.best_trials[0].params.map((param) => ( - - {param.name} {param.value} - - ))} - + + number={studyDetail.best_trials[0].number} + + + trial_id={studyDetail.best_trials[0].trial_id} + + + Params = [ + {studyDetail.best_trials[0].params + .map((p) => `${p.name}: ${p.value}`) + .join(", ")} + ] + + + Intermediate Values = [ + {studyDetail.best_trials[0].intermediate_values + .map((p) => `${p.step}: ${p.value}`) + .join(", ")} + ] + + + User Attributes = [ + {studyDetail.best_trials[0].user_attrs + .map((p) => `${p.key}: ${p.value}`) + .join(", ")} + ] + )} - {studyDetail !== null && studyDetail.best_trials.length > 1 && ( + {studyDetail !== null && studyDetail.directions.length > 1 && ( <> - Best Trials + Best Trials ({studyDetail.best_trials.length} trials) + {studyDetail.best_trials.map((trial, i) => ( + + + + Trial number={trial.number} (trial_id= + {trial.trial_id}) + + + Objective Values = [{trial.values?.join(", ")}] + + + Params = [ + {trial.params + .map((p) => `${p.name}: ${p.value}`) + .join(", ")} + ] + + + + ))} )} - - + + - - Hyperparameter Importance - - - - - - Hyperparameter Relationships @@ -214,25 +255,35 @@ export const StudyDetailBeta: FC<{ + + Empirical Distribution of the Objective Value + + + + + + ) - } else if (page === "trials") { + } else if (page === "trialTable") { content = ( - + ) - } else if (page === "note") { - content = - studyDetail !== null ? ( - - ) : null + } else if (page === "trialList") { + content = + } else if (page === "note" && studyDetail !== null) { + content = ( + + ) } const toolbar = ( diff --git a/optuna_dashboard/ts/components/StudyList.tsx b/optuna_dashboard/ts/components/StudyList.tsx index 11c2696e..f749ad16 100644 --- a/optuna_dashboard/ts/components/StudyList.tsx +++ b/optuna_dashboard/ts/components/StudyList.tsx @@ -23,7 +23,6 @@ import { DebouncedInputTextField } from "./Debounce" import { studySummariesState } from "../state" import Brightness7Icon from "@mui/icons-material/Brightness7" import Brightness4Icon from "@mui/icons-material/Brightness4" -import { useSnackbar } from "notistack" import { useDeleteStudyDialog } from "./DeleteStudyDialog" import { useCreateStudyDialog } from "./CreateStudyDialog" @@ -31,7 +30,6 @@ export const StudyList: FC<{ toggleColorMode: () => void }> = ({ toggleColorMode }) => { const theme = useTheme() - const { enqueueSnackbar } = useSnackbar() const [studyFilterText, setStudyFilterText] = React.useState("") const studyFilter = (row: StudySummary) => { @@ -150,15 +148,6 @@ export const StudyList: FC<{ ) } - const sayThankYouForBetaUsers = () => { - enqueueSnackbar( - `Thanks for testing our beta UI. Share your feedback via a GitHub issue.`, - { - variant: "success", - } - ) - } - return ( <> @@ -225,11 +214,7 @@ export const StudyList: FC<{ {`We would appreciate your feedback on our beta UI. Click `} - + here {" to try it out and share your thoughts."} diff --git a/optuna_dashboard/ts/components/StudyListBeta.tsx b/optuna_dashboard/ts/components/StudyListBeta.tsx index 08977813..02175a19 100644 --- a/optuna_dashboard/ts/components/StudyListBeta.tsx +++ b/optuna_dashboard/ts/components/StudyListBeta.tsx @@ -19,6 +19,7 @@ import { } from "@mui/material" import { Delete, Refresh, Search } from "@mui/icons-material" import SortIcon from "@mui/icons-material/Sort" +import HomeIcon from "@mui/icons-material/Home" import AddBoxIcon from "@mui/icons-material/AddBox" import { actionCreator } from "../action" @@ -101,11 +102,7 @@ export const StudyListBeta: FC<{ ) - const toolbar = ( - - Optuna Dashboard (Beta ver.) - - ) + const toolbar = return ( @@ -153,7 +150,7 @@ export const StudyListBeta: FC<{ }} sx={{ marginRight: theme.spacing(2), minWidth: "120px" }} > - Refresh + Reload