wassname/optuna-dashboard
1
0
Fork 0
mirror of https://github.com/wassname/optuna-dashboard.git synced 2026-09-10 12:23:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:optuna/optuna-dashboard into preferential-gp

Browse Source
This commit is contained in:
Contramundum
2023-08-22 14:10:36 +09:00
parent 7ea5fb639d f1259305b5
commit 2ec64ed19f
25 changed files with 1790 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.eslintignore
+2
View File
@@ -0,0 +1,2 @@
venv/**/*.ts
venv/**/*.js
.readthedocs.yaml
+18
View File
@@ -0,0 +1,18 @@
version: 2
build:
os: ubuntu-22.04
tools:
python: "3.11"
sphinx:
configuration: docs/conf.py
formats: all
python:
install:
- method: pip
path: .
extra_requirements:
- docs
optuna_dashboard/_app.py
+30 -12
View File
@@ -38,6 +38,9 @@ from ._storage_url import get_storage
from .artifact._backend import delete_all_artifacts
from .artifact._backend import register_artifact_route
from .artifact._backend_to_store import to_artifact_store
from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY
from .preferential._study import get_best_trials as get_best_preferential_trials
from .preferential._system_attrs import report_preferences
if typing.TYPE_CHECKING:
@@ -187,8 +190,12 @@ def create_app(
return {"reason": f"study_id={study_id} is not found"}
trials = get_trials(storage, study_id)
system_attrs = getattr(summary, "system_attrs", {})
is_preferential = system_attrs.get(_SYSTEM_ATTR_PREFERENTIAL_STUDY, False)
# TODO(c-bata): Cache best_trials
if len(summary.directions) == 1:
if is_preferential:
best_trials = get_best_preferential_trials(study_id, storage)
elif len(summary.directions) == 1:
if len([t for t in trials if t.state == TrialState.COMPLETE]) == 0:
best_trials = []
else:
@@ -255,6 +262,25 @@ def create_app(
response.status = 204 # No content
return {}
@app.post("/api/studies/<study_id:int>/preference")
@json_api_view
def post_preference(study_id: int) -> dict[str, Any]:
try:
best_trials = [int(d) for d in request.json.get("best_trials", [])]
worst_trials = [int(d) for d in request.json.get("worst_trials", [])]
except ValueError:
response.status = 400
return {"reason": "best_trials and worst_trials must be an array of integers."}
if len(best_trials) == 0 or len(worst_trials) == 0:
response.status = 400 # Bad request
return {"reason": "You need to set best_trials and worst_trials"}
preferences = [(best, worst) for best in best_trials for worst in worst_trials]
report_preferences(study_id, storage, preferences)
response.status = 204
return {}
@app.post("/api/trials/<trial_id:int>/tell")
@json_api_view
def tell_trial(trial_id: int) -> dict[str, Any]:
@@ -284,11 +310,7 @@ def create_app(
response.status = 400 # Bad request
return {"reason": "values attribute must be an array of numbers"}
try:
storage.set_trial_state_values(trial_id, state, values)
except Exception as e:
response.status = 500
return {"reason": f"Internal server error: {e}"}
storage.set_trial_state_values(trial_id, state, values)
response.status = 204
return {}
@@ -301,12 +323,8 @@ def create_app(
response.status = 400 # Bad request
return {"reason": "user_attrs must be specified."}
try:
for key, val in user_attrs.items():
storage.set_trial_user_attr(trial_id, key, val)
except Exception as e:
response.status = 500
return {"reason": f"Internal server error: {e}"}
for key, val in user_attrs.items():
storage.set_trial_user_attr(trial_id, key, val)
response.status = 204
return {}
optuna_dashboard/_serializer.py
+5 -4
View File
@@ -15,6 +15,7 @@ from . import _note as note
from ._form_widget import get_form_widgets_json
from ._named_objectives import get_objective_names
from .artifact._backend import list_trial_artifacts
from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY
if TYPE_CHECKING:
@@ -107,7 +108,9 @@ def serialize_study_summary(summary: StudySummary) -> dict[str, Any]:
"study_name": summary.study_name,
"directions": [d.name.lower() for d in summary.directions],
"user_attrs": serialize_attrs(summary.user_attrs),
"system_attrs": serialize_attrs(getattr(summary, "system_attrs", {})),
"is_preferential": getattr(summary, "_system_attrs", {}).get(
_SYSTEM_ATTR_PREFERENTIAL_STUDY, False
),
}
if summary.datetime_start is not None:
@@ -144,6 +147,7 @@ def serialize_study_detail(
serialized["union_user_attrs"] = [{"key": a[0], "sortable": a[1]} for a in union_user_attrs]
serialized["has_intermediate_values"] = has_intermediate_values
serialized["note"] = note.get_note_from_system_attrs(system_attrs, None)
serialized["is_preferential"] = system_attrs.get(_SYSTEM_ATTR_PREFERENTIAL_STUDY, False)
objective_names = get_objective_names(system_attrs)
if objective_names:
serialized["objective_names"] = objective_names
@@ -183,9 +187,6 @@ def serialize_frozen_trial(
for param_name in fixed_params
],
"user_attrs": serialize_attrs(trial.user_attrs),
"system_attrs": serialize_attrs(
{k: trial_system_attrs[k] for k in trial_system_attrs if not k.startswith("dashboard")}
),
"note": note.get_note_from_system_attrs(study_system_attrs, trial._trial_id),
"artifacts": list_trial_artifacts(study_system_attrs, trial),
"constraints": trial_system_attrs.get(CONSTRAINTS_KEY, []),
optuna_dashboard/preferential/_study.py
+24 -12
View File
@@ -31,16 +31,7 @@ class PreferentialStudy:
@property
def best_trials(self) -> list[FrozenTrial]:
ready_trials = [
t
for t in self._study.get_trials(
deepcopy=False, states=(TrialState.COMPLETE, TrialState.RUNNING)
)
if t.system_attrs.get(_SYSTEM_ATTR_COMPARISON_READY) is True
]
preferences = get_preferences(self._study, deepcopy=False)
worse_numbers = {worse.number for _, worse in preferences}
return [copy.deepcopy(t) for t in ready_trials if t.number not in worse_numbers]
return get_best_trials(self._study._study_id, self._study._storage)
@property
def study_name(self) -> str:
@@ -80,10 +71,16 @@ class PreferentialStudy:
if not isinstance(worse_trials, list):
worse_trials = [worse_trials]
report_preferences(self._study, [(b, w) for b in better_trials for w in worse_trials])
report_preferences(
self._study._study_id,
self._study._storage,
[(b.number, w.number) for b in better_trials for w in worse_trials],
)
def get_preferences(self, *, deepcopy: bool = True) -> list[tuple[FrozenTrial, FrozenTrial]]:
return get_preferences(self._study, deepcopy=deepcopy)
trials = self._study.get_trials(deepcopy=deepcopy)
preferences = get_preferences(self._study._study_id, self._study._storage)
return [(trials[better], trials[worse]) for (better, worse) in preferences]
def set_user_attr(self, key: str, value: Any) -> None:
self._study.set_user_attr(key, value)
@@ -101,6 +98,21 @@ class PreferentialStudy:
storage.set_trial_system_attr(trial_id, _SYSTEM_ATTR_COMPARISON_READY, True)
def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]:
ready_trials = [
t
for t in storage.get_all_trials(
study_id,
deepcopy=False,
states=(TrialState.COMPLETE, TrialState.RUNNING),
)
if t.system_attrs.get(_SYSTEM_ATTR_COMPARISON_READY) is True
]
preferences = get_preferences(study_id, storage)
worse_numbers = {worse for _, worse in preferences}
return [copy.deepcopy(t) for t in ready_trials if t.number not in worse_numbers]
def create_study(
*,
storage: str | optuna.storages.BaseStorage | None = None,
optuna_dashboard/preferential/_system_attrs.py
+24 -24
View File
@@ -2,45 +2,45 @@ from __future__ import annotations
import uuid
import optuna
from optuna.trial import FrozenTrial
from optuna.storages import BaseStorage
from optuna.trial import TrialState
from .._storage import get_study_summary
_SYSTEM_ATTR_PREFIX_PREFERENCE = "preference:values"
def report_preferences(
study: optuna.Study,
preferences: list[tuple[FrozenTrial, FrozenTrial]],
study_id: int,
storage: BaseStorage,
preferences: list[tuple[int, int]],
) -> None:
key = _SYSTEM_ATTR_PREFIX_PREFERENCE + str(uuid.uuid4())
study._storage.set_study_system_attr(
study_id=study._study_id,
storage.set_study_system_attr(
study_id=study_id,
key=key,
value=[(better.number, worse.number) for better, worse in preferences],
value=preferences,
)
values = [0 for _ in study.directions]
for better, worse in preferences:
for t in (better, worse):
study.tell(
t.number,
values=values,
state=TrialState.COMPLETE,
skip_if_finished=True,
)
trials = storage.get_all_trials(study_id, deepcopy=False)
directions = storage.get_study_directions(study_id)
values = [0 for _ in directions]
updated_trials = {num for tpl in preferences for num in tpl}
for number in updated_trials:
trial_id = trials[number]._trial_id
if trials[number].state != TrialState.COMPLETE:
storage.set_trial_state_values(trial_id, TrialState.COMPLETE, values)
def get_preferences(
study: optuna.Study,
*,
deepcopy: bool = True,
) -> list[tuple[FrozenTrial, FrozenTrial]]:
study_id: int,
storage: BaseStorage,
) -> list[tuple[int, int]]:
preferences: list[tuple[int, int]] = []
for k, v in study.system_attrs.items():
summary = get_study_summary(storage, study_id)
system_attrs = getattr(summary, "system_attrs", {})
for k, v in system_attrs.items():
if not k.startswith(_SYSTEM_ATTR_PREFIX_PREFERENCE):
continue
preferences.extend(v) # type: ignore
trials = study.get_trials(deepcopy=deepcopy)
return [(trials[better], trials[worse]) for (better, worse) in preferences]
return preferences
optuna_dashboard/ts/action.ts
+17
View File
@@ -14,6 +14,7 @@ import {
uploadArtifactAPI,
getMetaInfoAPI,
deleteArtifactAPI,
reportPreferenceAPI,
} from "./apiClient"
import {
graphVisibilityState,
@@ -582,6 +583,21 @@ export const actionCreator = () => {
console.log(err)
})
}
const updatePreference = (
study_id: number,
best_trials: number[],
worst_trials: number[]
) => {
reportPreferenceAPI(study_id, best_trials, worst_trials).catch((err) => {
const reason = err.response?.data.reason
enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, {
variant: "error",
})
console.log(err)
})
}
return {
updateAPIMeta,
updateStudyDetail,
@@ -601,6 +617,7 @@ export const actionCreator = () => {
makeTrialComplete,
makeTrialFail,
saveTrialUserAttrs,
updatePreference,
}
}
optuna_dashboard/ts/apiClient.ts
+23 -8
View File
@@ -27,7 +27,6 @@ interface TrialResponse {
param_external_value: string
}[]
user_attrs: Attribute[]
system_attrs: Attribute[]
note: Note
artifacts: Artifact[]
constraints: number[]
@@ -50,7 +49,6 @@ const convertTrialResponse = (res: TrialResponse): Trial => {
params: res.params,
fixed_params: res.fixed_params,
user_attrs: res.user_attrs,
system_attrs: res.system_attrs,
note: res.note,
artifacts: res.artifacts,
constraints: res.constraints,
@@ -68,6 +66,7 @@ interface StudyDetailResponse {
union_user_attrs: AttributeSpec[]
has_intermediate_values: boolean
note: Note
is_preferential: boolean
objective_names?: string[]
form_widgets?: FormWidgets
}
@@ -103,6 +102,7 @@ export const getStudyDetailAPI = (
note: res.data.note,
objective_names: res.data.objective_names,
form_widgets: res.data.form_widgets,
is_preferential: res.data.is_preferential,
}
})
}
@@ -113,7 +113,7 @@ interface StudySummariesResponse {
study_name: string
directions: StudyDirection[]
user_attrs: Attribute[]
system_attrs: Attribute[]
is_preferential: boolean
datetime_start?: string
}[]
}
@@ -128,7 +128,7 @@ export const getStudySummariesAPI = (): Promise<StudySummary[]> => {
study_name: study.study_name,
directions: study.directions,
user_attrs: study.user_attrs,
system_attrs: study.system_attrs,
is_preferential: study.is_preferential,
datetime_start: study.datetime_start
? new Date(study.datetime_start)
: undefined,
@@ -143,7 +143,7 @@ interface CreateNewStudyResponse {
study_name: string
directions: StudyDirection[]
user_attrs: Attribute[]
system_attrs: Attribute[]
is_preferential: boolean
datetime_start?: string
}
}
@@ -165,7 +165,7 @@ export const createNewStudyAPI = (
directions: study_summary.directions,
// best_trial: undefined,
user_attrs: study_summary.user_attrs,
system_attrs: study_summary.system_attrs,
is_preferential: study_summary.is_preferential,
datetime_start: study_summary.datetime_start
? new Date(study_summary.datetime_start)
: undefined,
@@ -184,7 +184,7 @@ type RenameStudyResponse = {
study_name: string
directions: StudyDirection[]
user_attrs: Attribute[]
system_attrs: Attribute[]
is_prefential: boolean
datetime_start?: string
}
@@ -202,7 +202,7 @@ export const renameStudyAPI = (
study_name: res.data.study_name,
directions: res.data.directions,
user_attrs: res.data.user_attrs,
system_attrs: res.data.system_attrs,
is_preferential: res.data.is_prefential,
datetime_start: res.data.datetime_start
? new Date(res.data.datetime_start)
: undefined,
@@ -309,3 +309,18 @@ export const getParamImportances = (
return res.data.param_importances
})
}
export const reportPreferenceAPI = (
studyId: number,
best_trials: number[],
worst_trials: number[]
): Promise<void> => {
return axiosInstance
.post<void>(`/api/studies/${studyId}/preference`, {
best_trials: best_trials,
worst_trials: worst_trials,
})
.then(() => {
return
})
}
optuna_dashboard/ts/components/App.tsx
+1 -10
View File
@@ -69,15 +69,6 @@ export const App: FC = () => {
/>
}
/>
<Route
path={URL_PREFIX + "/studies/:studyId/trials"}
element={
<StudyDetail
toggleColorMode={toggleColorMode}
page={"trialList"}
/>
}
/>
<Route
path={URL_PREFIX + "/studies/:studyId/trialTable"}
element={
@@ -101,7 +92,7 @@ export const App: FC = () => {
element={
<StudyDetail
toggleColorMode={toggleColorMode}
page={"history"}
page={"top"}
/>
}
/>
optuna_dashboard/ts/components/AppDrawer.tsx
+36 -24
View File
@@ -14,7 +14,12 @@ import ListItem from "@mui/material/ListItem"
import ListItemButton from "@mui/material/ListItemButton"
import ListItemIcon from "@mui/material/ListItemIcon"
import ListItemText from "@mui/material/ListItemText"
import { drawerOpenState, reloadIntervalState } from "../state"
import {
drawerOpenState,
reloadIntervalState,
useStudyDetailValue,
useStudySummaryValue,
} from "../state"
import { Link } from "react-router-dom"
import AutoGraphIcon from "@mui/icons-material/AutoGraph"
import ViewListIcon from "@mui/icons-material/ViewList"
@@ -28,17 +33,13 @@ import MenuIcon from "@mui/icons-material/Menu"
import GitHubIcon from "@mui/icons-material/GitHub"
import OpenInNewIcon from "@mui/icons-material/OpenInNew"
import QueryStatsIcon from "@mui/icons-material/QueryStats"
import ThumbUpAltIcon from "@mui/icons-material/ThumbUpAlt"
import { Switch } from "@mui/material"
import { actionCreator } from "../action"
const drawerWidth = 240
export type PageId =
| "history"
| "analytics"
| "trialTable"
| "trialList"
| "note"
export type PageId = "top" | "analytics" | "trialTable" | "trialList" | "note"
const openedMixin = (theme: Theme): CSSObject => ({
width: drawerWidth,
@@ -120,6 +121,12 @@ export const AppDrawer: FC<{
const action = actionCreator()
const [open, setOpen] = useRecoilState<boolean>(drawerOpenState)
const reloadInterval = useRecoilValue<number>(reloadIntervalState)
const studyDetail =
studyId !== undefined ? useStudyDetailValue(studyId) : null
const studySummary =
studyId !== undefined ? useStudySummaryValue(studyId) : null
const isPreferential =
studyDetail?.is_preferential ?? studySummary?.is_preferential ?? false
const styleListItem = {
display: "block",
@@ -181,32 +188,37 @@ export const AppDrawer: FC<{
<Divider />
{studyId !== undefined && page && (
<List>
<ListItem key="History" disablePadding sx={styleListItem}>
<ListItem key="Top" disablePadding sx={styleListItem}>
<ListItemButton
component={Link}
to={`${URL_PREFIX}/studies/${studyId}`}
sx={styleListItemButton}
selected={page === "history"}
selected={page === "top"}
>
<ListItemIcon sx={styleListItemIcon}>
<AutoGraphIcon />
{isPreferential ? <ThumbUpAltIcon /> : <AutoGraphIcon />}
</ListItemIcon>
<ListItemText primary="History" sx={styleListItemText} />
</ListItemButton>
</ListItem>
<ListItem key="Analytics" disablePadding sx={styleListItem}>
<ListItemButton
component={Link}
to={`${URL_PREFIX}/studies/${studyId}/analytics`}
sx={styleListItemButton}
selected={page === "analytics"}
>
<ListItemIcon sx={styleListItemIcon}>
<QueryStatsIcon />
</ListItemIcon>
<ListItemText primary="Analytics" sx={styleListItemText} />
<ListItemText
primary={isPreferential ? "HumanInTheLoop" : "History"}
sx={styleListItemText}
/>
</ListItemButton>
</ListItem>
{!isPreferential && (
<ListItem key="Analytics" disablePadding sx={styleListItem}>
<ListItemButton
component={Link}
to={`${URL_PREFIX}/studies/${studyId}/analytics`}
sx={styleListItemButton}
selected={page === "analytics"}
>
<ListItemIcon sx={styleListItemIcon}>
<QueryStatsIcon />
</ListItemIcon>
<ListItemText primary="Analytics" sx={styleListItemText} />
</ListItemButton>
</ListItem>
)}
<ListItem key="TableList" disablePadding sx={styleListItem}>
<ListItemButton
component={Link}
optuna_dashboard/ts/components/GraphHistory.tsx
+1 -1
View File
@@ -206,7 +206,7 @@ const plotHistory = (
title: xAxis === "number" ? "Trial" : "Time",
type: xAxis === "number" ? "linear" : "date",
},
showlegend: true,
showlegend: historyPlotInfos.length === 1 ? false : true,
template: mode === "dark" ? plotlyDarkTemplate : {},
}
optuna_dashboard/ts/components/PreferentialTrials.tsx
+120
View File
@@ -0,0 +1,120 @@
import React, { FC, useState } from "react"
import { Typography, Box, Button, useTheme } from "@mui/material"
import { TrialNote } from "./Note"
import { actionCreator } from "../action"
const PreferentialTrial: FC<{
trial?: Trial
studyDetail: StudyDetail
hideTrial: () => void
}> = ({ trial, studyDetail, hideTrial }) => {
const theme = useTheme()
const action = actionCreator()
const trialWidth = 500
if (trial == undefined) {
return <Box width={trialWidth}></Box>
}
return (
<Box sx={{ width: trialWidth, padding: theme.spacing(2, 2, 0, 2) }}>
<Typography
variant="h4"
sx={{
marginBottom: theme.spacing(2),
fontWeight: theme.typography.fontWeightBold,
}}
>
Trial {trial.number} (trial_id={trial.trial_id})
</Typography>
<Button
variant="outlined"
onClick={() => {
hideTrial()
const best_trials = studyDetail.best_trials
.map((t) => t.number)
.filter((t) => t !== trial.number)
action.updatePreference(trial.study_id, best_trials, [trial.number])
}}
>
Worst
</Button>
<TrialNote
studyId={trial.study_id}
trialId={trial.trial_id}
latestNote={trial.note}
cardSx={{ marginBottom: theme.spacing(2) }}
/>
</Box>
)
}
type DisplayTrials = {
numbers: number[]
last_number: number
}
export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({
studyDetail,
}) => {
if (studyDetail === null || !studyDetail.is_preferential) {
return null
}
const [displayTrials, setDisplayTrials] = useState<DisplayTrials>({
numbers: studyDetail.best_trials.map((t) => t.number),
last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1),
})
const new_trails = studyDetail.best_trials.filter(
(t) =>
displayTrials.last_number < t.number &&
displayTrials.numbers.find((n) => n === t.number) === undefined
)
if (new_trails.length > 0) {
setDisplayTrials((display) => {
const numbers = [...display.numbers]
new_trails.map((t) => {
const index = numbers.findIndex((n) => n === -1)
if (index === -1) {
numbers.push(t.number)
} else {
numbers[index] = t.number
}
})
return {
numbers: numbers,
last_number: Math.max(...numbers, -1),
}
})
}
const hideTrial = (num: number) => {
setDisplayTrials((display) => {
const index = display.numbers.findIndex((n) => n === num)
if (index === -1) {
return display
}
const numbers = [...displayTrials.numbers]
numbers[index] = -1
return {
numbers: numbers,
last_number: display.last_number,
}
})
}
return (
<Box sx={{ display: "flex", flexDirection: "row", flexWrap: "wrap" }}>
{displayTrials.numbers.map((t, index) => (
<PreferentialTrial
key={index}
trial={studyDetail.best_trials.find((trial) => trial.number === t)}
studyDetail={studyDetail}
hideTrial={() => {
hideTrial(t)
}}
/>
))}
</Box>
)
}
optuna_dashboard/ts/components/StudyDetail.tsx
+21 -7
View File
@@ -19,6 +19,7 @@ import {
reloadIntervalState,
useStudyDetailValue,
useStudyName,
useStudySummaryValue,
} from "../state"
import { TrialTable } from "./TrialTable"
import { AppDrawer, PageId } from "./AppDrawer"
@@ -28,6 +29,7 @@ import { GraphSlice } from "./GraphSlice"
import { GraphEdf } from "./GraphEdf"
import { TrialList } from "./TrialList"
import { StudyHistory } from "./StudyHistory"
import { PreferentialTrials } from "./PreferentialTrials"
interface ParamTypes {
studyId: string
@@ -47,8 +49,11 @@ export const StudyDetail: FC<{
const action = actionCreator()
const studyId = useURLVars()
const studyDetail = useStudyDetailValue(studyId)
const studySummary = useStudySummaryValue(studyId)
const reloadInterval = useRecoilValue<number>(reloadIntervalState)
const studyName = useStudyName(studyId)
const isPreferential =
studySummary?.is_preferential ?? studyDetail?.is_preferential ?? false
const title =
studyName !== null ? `${studyName} (id=${studyId})` : `Study #${studyId}`
@@ -67,11 +72,16 @@ export const StudyDetail: FC<{
let interval = reloadInterval * 1000
// For Human-in-the-loop Optimization, the interval is set to 2 seconds
// when the number of trials is small and the page is "trialList".
if (page === "trialList" && nTrials < 100) {
interval = 2000
} else if (page === "trialList" && nTrials < 500) {
interval = 5000
// when the number of trials is small, and the page is "trialList" or top page of preferential.
if (
(!isPreferential && page === "trialList") ||
(isPreferential && page === "top")
) {
if (nTrials < 100) {
interval = 2000
} else if (nTrials < 500) {
interval = 5000
}
}
const intervalId = setInterval(function () {
@@ -81,8 +91,12 @@ export const StudyDetail: FC<{
}, [reloadInterval, studyDetail, page])
let content = null
if (page === "history") {
content = <StudyHistory studyId={studyId} />
if (page === "top") {
content = isPreferential ? (
<PreferentialTrials studyDetail={studyDetail} />
) : (
<StudyHistory studyId={studyId} />
)
} else if (page === "analytics") {
content = (
<Box sx={{ display: "flex", width: "100%", flexDirection: "column" }}>
optuna_dashboard/ts/components/StudyList.tsx
+6 -4
View File
@@ -215,10 +215,12 @@ export const StudyList: FC<{
color="text.secondary"
component="div"
>
{"Direction: " +
study.directions
.map((d) => d.toString().toUpperCase())
.join(", ")}
{study.is_preferential
? "Preferential Optimization"
: "Direction: " +
study.directions
.map((d) => d.toString().toUpperCase())
.join(", ")}
</Typography>
</CardContent>
</CardActionArea>
optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx
+111
View File
@@ -0,0 +1,111 @@
import * as THREE from "three"
import React, { useEffect, useState } from "react"
import { Canvas } from "@react-three/fiber"
import { GizmoHelper, GizmoViewport, OrbitControls } from "@react-three/drei"
import { STLLoader } from "three/examples/jsm/loaders/STLLoader"
import { Rhino3dmLoader } from "three/examples/jsm/loaders/3DMLoader"
import { PerspectiveCamera } from "three"
interface ThreejsArtifactViewerProps {
src: string
width: string
height: string
hasGizmo: boolean
filetype: string | undefined
}
const CustomGizmoHelper: React.FC = () => {
return (
<GizmoHelper alignment="bottom-right" margin={[80, 80]}>
<GizmoViewport
axisColors={["red", "green", "skyblue"]}
labelColor="black"
/>
</GizmoHelper>
)
}
const calculateBoundingBox = (geometries: THREE.BufferGeometry[]) => {
const boundingBox = new THREE.Box3()
geometries.forEach((geometry) => {
const mesh = new THREE.Mesh(geometry)
boundingBox.expandByObject(mesh)
})
return boundingBox
}
export const ThreejsArtifactViewer: React.FC<ThreejsArtifactViewerProps> = (
props
) => {
const [geometry, setGeometry] = useState<THREE.BufferGeometry[]>([])
const [modelSize, setModelSize] = useState<THREE.Vector3>(
new THREE.Vector3(10, 10, 10)
)
const [cameraSettings, setCameraSettings] = useState<PerspectiveCamera>(
new THREE.PerspectiveCamera()
)
const handleLoadedGeometries = (geometries: THREE.BufferGeometry[]) => {
setGeometry(geometries)
const boundingBox = calculateBoundingBox(geometries)
if (boundingBox !== null) {
const size = boundingBox.getSize(new THREE.Vector3())
setModelSize(size)
}
}
useEffect(() => {
if ("stl" === props.filetype) {
const stlLoader = new STLLoader()
stlLoader.load(props.src, (stlGeometries: THREE.BufferGeometry) => {
if (stlGeometries) {
handleLoadedGeometries([stlGeometries])
}
})
} else if ("3dm" === props.filetype) {
const loader = new Rhino3dmLoader()
loader.setLibraryPath("https://cdn.jsdelivr.net/npm/rhino3dm@7.15.0/")
loader.load(props.src, (object: THREE.Object3D) => {
const meshes = object.children as THREE.Mesh[]
const rhinoGeometries = meshes.map((mesh) => mesh.geometry)
if (rhinoGeometries.length > 0) {
rhinoGeometries.forEach((rhinoGeometry) => {
rhinoGeometry.rotateX(-Math.PI / 4)
})
handleLoadedGeometries(rhinoGeometries)
}
})
}
const maxModelSize = Math.max(modelSize.x, modelSize.y, modelSize.z)
const cameraSet = new THREE.PerspectiveCamera(
modelSize
? Math.min(
45,
Math.atan(modelSize.y / modelSize.z) * (180 / Math.PI) * 2
)
: 45,
window.innerWidth / window.innerHeight
)
cameraSet.position.set(maxModelSize * 2, maxModelSize * 2, maxModelSize * 2)
setCameraSettings(cameraSet)
}, [])
return (
<Canvas
camera={cameraSettings}
style={{ width: props.width, height: props.height }}
>
<ambientLight />
<OrbitControls />
<gridHelper args={[Math.max(modelSize?.x, modelSize?.y) * 5]} />
{props.hasGizmo && <CustomGizmoHelper />}
<axesHelper />
{geometry.length > 0 &&
geometry.map((geo, index) => (
<mesh key={index} geometry={geo}>
<meshNormalMaterial />
</mesh>
))}
</Canvas>
)
}
optuna_dashboard/ts/components/TrialList.tsx
+132
View File
@@ -20,6 +20,7 @@ import {
CardContent,
CardMedia,
CardActionArea,
Modal,
} from "@mui/material"
import Chip from "@mui/material/Chip"
import Divider from "@mui/material/Divider"
@@ -34,6 +35,7 @@ import CheckBoxIcon from "@mui/icons-material/CheckBox"
import UploadFileIcon from "@mui/icons-material/UploadFile"
import DownloadIcon from "@mui/icons-material/Download"
import DeleteIcon from "@mui/icons-material/Delete"
import FullscreenIcon from "@mui/icons-material/Fullscreen"
import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile"
import StopCircleIcon from "@mui/icons-material/StopCircle"
@@ -45,6 +47,7 @@ import { artifactIsAvailable } from "../state"
import { actionCreator } from "../action"
import { useDeleteArtifactDialog } from "./DeleteArtifactDialog"
import { TrialFormWidgets } from "./TrialFormWidgets"
import { ThreejsArtifactViewer } from "./ThreejsArtifactViewer"
const states: TrialState[] = [
"Complete",
@@ -327,6 +330,9 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => {
const [openDeleteArtifactDialog, renderDeleteArtifactDialog] =
useDeleteArtifactDialog()
const [dragOver, setDragOver] = useState<boolean>(false)
const [open3dModelViewer, setOpen3dModelViewer] = useState<{
[key: string]: boolean
}>({})
const width = "200px"
const height = "150px"
@@ -366,6 +372,7 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => {
e.dataTransfer.dropEffect = "copy"
setDragOver(false)
}
return (
<>
<Typography
@@ -437,6 +444,131 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => {
</CardContent>
</Card>
)
} else if (
a.filename.endsWith(".stl") ||
a.filename.endsWith(".3dm")
) {
return (
<Card
key={a.artifact_id}
sx={{
marginBottom: theme.spacing(2),
display: "flex",
flexDirection: "column",
width: width,
minHeight: "100%",
margin: theme.spacing(0, 1, 1, 0),
}}
>
<Box
sx={{
flexGrow: 1,
display: "flex",
justifyContent: "center",
alignItems: "center",
}}
>
<ThreejsArtifactViewer
src={`/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}`}
width={width}
height={height}
hasGizmo={false}
filetype={a.filename.split(".").pop()}
/>
</Box>
<CardContent
sx={{
display: "flex",
flexDirection: "row",
padding: `${theme.spacing(1)} !important`,
}}
>
<Typography
sx={{
p: theme.spacing(0.5, 0),
flexGrow: 1,
wordWrap: "break-word",
maxWidth: `calc(100% - ${theme.spacing(12)})`,
}}
>
{a.filename}
</Typography>
<IconButton
aria-label="show artifact 3d model"
size="small"
color="inherit"
sx={{ margin: "auto 0" }}
onClick={() => {
setOpen3dModelViewer(() => {
const obj = { ...open3dModelViewer }
obj[a.artifact_id] = true
return obj
})
}}
>
<FullscreenIcon />
</IconButton>
<Modal
open={
a.artifact_id in open3dModelViewer
? open3dModelViewer[a.artifact_id]
: false
}
onClose={() => {
setOpen3dModelViewer(() => {
const obj = { ...open3dModelViewer }
obj[a.artifact_id] = false
return obj
})
}}
>
<Box
sx={{
position: "absolute",
top: "50%",
left: "50%",
transform: "translate(-50%, -50%)",
bgcolor: "background.paper",
borderRadius: "15px",
}}
>
<ThreejsArtifactViewer
src={`/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}`}
width={`${innerWidth * 0.8}px`}
height={`${innerHeight * 0.8}px`}
hasGizmo={true}
filetype={a.filename.split(".").pop()}
/>
</Box>
</Modal>
<IconButton
aria-label="delete artifact"
size="small"
color="inherit"
sx={{ margin: "auto 0" }}
onClick={() => {
openDeleteArtifactDialog(
trial.study_id,
trial.trial_id,
a
)
}}
>
<DeleteIcon />
</IconButton>
<IconButton
aria-label="download artifact"
size="small"
color="inherit"
sx={{ margin: "auto 0" }}
download={a.filename}
href={`/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}`}
>
<DownloadIcon />
</IconButton>
</CardContent>
</Card>
)
} else if (a.mimetype.startsWith("audio")) {
return (
<Card
optuna_dashboard/ts/types/index.d.ts
Vendored
+2 -2
View File
@@ -111,7 +111,6 @@ type Trial = {
param_external_value: string
}[]
user_attrs: Attribute[]
system_attrs: Attribute[]
constraints: number[]
note: Note
artifacts: Artifact[]
@@ -122,7 +121,7 @@ type StudySummary = {
study_name: string
directions: StudyDirection[]
user_attrs: Attribute[]
system_attrs: Attribute[]
is_preferential: boolean
datetime_start?: Date
}
@@ -194,6 +193,7 @@ type StudyDetail = {
union_user_attrs: AttributeSpec[]
has_intermediate_values: boolean
note: Note
is_preferential: boolean
objective_names?: string[]
form_widgets?: FormWidgets
}
package-lock.json
Generated
+1076 -51
View File
File diff suppressed because it is too large Load Diff
package.json
+5 -2
View File
@@ -23,6 +23,9 @@
"@mui/icons-material": "^5.11.6",
"@mui/lab": "^5.0.0-alpha.128",
"@mui/material": "^5.12.1",
"@react-three/drei": "^9.80.0",
"@react-three/fiber": "^8.13.6",
"@types/three": "^0.154.0",
"axios": "^1.2.1",
"notistack": "^3.0.1",
"plotly.js-dist-min": "^2.22.0",
@@ -35,7 +38,8 @@
"rehype-mathjax": "^4.0.2",
"rehype-raw": "^6.1.1",
"remark-gfm": "^3.0.1",
"remark-math": "^5.1.1"
"remark-math": "^5.1.1",
"three": "^0.155.0"
},
"devDependencies": {
"@babel/core": "^7.14.3",
@@ -45,7 +49,6 @@
"@types/plotly.js": "^2.12.11",
"@types/react": "^18.0.26",
"@types/react-dom": "^18.0.10",
"@types/react-router-dom": "^5.3.3",
"@types/react-syntax-highlighter": "^15.5.5",
"@typescript-eslint/eslint-plugin": "^4.26.1",
"@typescript-eslint/parser": "^4.26.1",
pyproject.toml
+8
View File
@@ -33,6 +33,14 @@ dependencies = [
]
dynamic = ["version"]
[project.optional-dependencies]
docs = [
"boto3",
"streamlit",
"sphinx",
"sphinx_rtd_theme",
]
[project.scripts]
optuna-dashboard = "optuna_dashboard._cli:main"
python_tests/preferential/test_system_attrs.py
+7 -6
View File
@@ -17,12 +17,13 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli
study.ask()
study.ask()
assert len(get_preferences(study)) == 0
study_id = study._study_id
assert len(get_preferences(study_id, storage)) == 0
better, worse = study.trials[0], study.trials[1]
report_preferences(study, [(better, worse)])
assert len(get_preferences(study)) == 1
report_preferences(study_id, storage, [(better.number, worse.number)])
assert len(get_preferences(study_id, storage)) == 1
actual_better, actual_worse = get_preferences(study)[0]
assert actual_better.number == better.number
assert actual_worse.number == worse.number
actual_better, actual_worse = get_preferences(study_id, storage)[0]
assert actual_better == better.number
assert actual_worse == worse.number
python_tests/test_api.py
+52
View File
@@ -8,6 +8,7 @@ from optuna import get_all_study_summaries
from optuna.study import StudyDirection
from optuna_dashboard._app import create_app
from optuna_dashboard._app import create_new_study
from optuna_dashboard.preferential import create_study
from .wsgi_client import send_request
@@ -99,6 +100,57 @@ class APITestCase(TestCase):
)
self.assertEqual(status, 400)
def test_get_best_trials_of_preferential_study(self) -> None:
storage = optuna.storages.InMemoryStorage()
study = create_study(storage=storage)
for _ in range(3):
trial = study.ask()
study.mark_comparison_ready(trial)
study.report_preference(study.trials[0], study.trials[1])
app = create_app(storage)
study_id = study._study._study_id
status, _, body = send_request(
app,
f"/api/studies/{study_id}",
"GET",
content_type="application/json",
)
self.assertEqual(status, 200)
best_trials = json.loads(body)["best_trials"]
assert len(best_trials) == 2
assert best_trials[0]["number"] == 0
assert best_trials[1]["number"] == 2
def test_report_preference(self) -> None:
storage = optuna.storages.InMemoryStorage()
study = create_study(storage=storage)
for _ in range(3):
trial = study.ask()
study.mark_comparison_ready(trial)
app = create_app(storage)
study_id = study._study._study_id
status, _, _ = send_request(
app,
f"/api/studies/{study_id}/preference",
"POST",
body=json.dumps({"best_trials": [0, 2], "worst_trials": [1]}),
content_type="application/json",
)
self.assertEqual(status, 204)
preferences = study.get_preferences()
preferences.sort(key=lambda x: (x[0].number, x[1].number))
assert len(preferences) == 2
better, worse = preferences[0]
assert better.number == 0
assert worse.number == 1
better, worse = preferences[1]
assert better.number == 2
assert worse.number == 1
def test_create_study(self) -> None:
for name, directions, expected_status in [
("single-objective success", ["minimize"], 201),
python_tests/test_serializers.py
+57 -13
View File
@@ -1,19 +1,63 @@
from __future__ import annotations
from unittest import TestCase
import optuna
from optuna_dashboard._serializer import serialize_attrs
from optuna_dashboard._serializer import serialize_study_detail
from optuna_dashboard._serializer import serialize_study_summary
from optuna_dashboard._storage import get_study_summaries
from optuna_dashboard.preferential import create_study
class SerializeAttrsTestCase(TestCase):
def test_serialize_bytes(self) -> None:
serialized = serialize_attrs({"bytes": b"This is a bytes object."})
self.assertEqual(serialized[0]["value"], "<binary object>")
def test_serialize_bytes() -> None:
serialized = serialize_attrs({"bytes": b"This is a bytes object."})
assert serialized[0]["value"] == "<binary object>"
def test_serialize_dict(self) -> None:
serialized = serialize_attrs(
{
"key": {"foo": "bar"},
}
)
self.assertLessEqual(len(serialized), 1)
def test_serialize_dict() -> None:
serialized = serialize_attrs(
{
"key": {"foo": "bar"},
}
)
assert len(serialized) <= 1
def test_get_study_detail_is_preferential() -> None:
storage = optuna.storages.InMemoryStorage()
study = create_study(storage=storage)
study_summaries = get_study_summaries(storage)
assert len(study_summaries) == 1
study_summary = study_summaries[0]
study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False)
assert study_detail["is_preferential"]
def test_get_study_detail_is_not_preferential() -> None:
storage = optuna.storages.InMemoryStorage()
study = optuna.create_study(storage=storage)
study_summaries = get_study_summaries(storage)
assert len(study_summaries) == 1
study_summary = study_summaries[0]
study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False)
assert not study_detail["is_preferential"]
def test_get_study_summary_is_preferential() -> None:
storage = optuna.storages.InMemoryStorage()
create_study(storage=storage)
study_summaries = get_study_summaries(storage)
assert len(study_summaries) == 1
study_summary = serialize_study_summary(study_summaries[0])
assert study_summary["is_preferential"]
def test_get_study_summary_is_not_preferential() -> None:
storage = optuna.storages.InMemoryStorage()
optuna.create_study(storage=storage)
study_summaries = get_study_summaries(storage)
assert len(study_summaries) == 1
study_summary = serialize_study_summary(study_summaries[0])
assert not study_summary["is_preferential"]
python_tests/wsgi_client.py
+11
View File
@@ -6,12 +6,21 @@ from typing import Optional
from typing import Union
from bottle import Bottle
from optuna_dashboard._storage import trials_cache
from optuna_dashboard._storage import trials_cache_lock
from optuna_dashboard._storage import trials_last_fetched_at
if typing.TYPE_CHECKING:
from _typeshed.wsgi import WSGIEnvironment
def clear_inmemory_cache() -> None:
with trials_cache_lock:
trials_cache.clear()
trials_last_fetched_at.clear()
def create_wsgi_env(
path: str,
method: str,
@@ -66,6 +75,8 @@ def send_request(
headers = headers or {}
queries = queries or {}
env = create_wsgi_env(path, method, content_type, bytes_body, queries, headers)
clear_inmemory_cache()
response_body = b""
iterable_body = app(env, start_response)
for b in iterable_body:
requirements.txt
+1 -5
View File
@@ -18,8 +18,4 @@ boto3
moto[s3]
# visual regression tests
pytest-playwright
# docs
sphinx
sphinx_rtd_theme
pytest-playwright