wassname/optuna-dashboard
1
0
Fork 0
mirror of https://github.com/wassname/optuna-dashboard.git synced 2026-09-12 12:40:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

modify API

Browse Source
This commit is contained in:
moririn2528
2023-09-01 12:03:45 +09:00
parent 7e429266d6
commit b09e223e17
9 changed files with 109 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
optuna_dashboard/_app.py
+10 -14
View File
@@ -28,7 +28,6 @@ from ._bottle_util import json_api_view
from ._cached_extra_study_property import get_cached_extra_study_property
from ._importance import get_param_importance_from_trials_cache
from ._pareto_front import get_pareto_front_trials
from ._preferential_history import ChooseWorstHistory
from ._preferential_history import report_history
from ._rdb_migration import register_rdb_migration_route
from ._serializer import serialize_study_detail
@@ -281,22 +280,19 @@ def create_app(
if clicked == -1:
response.status = 400
return {"reason": "`clicked` should be specified."}
if mode == "ChooseWorst":
history = ChooseWorstHistory.create(
study_id,
storage,
{
"mode": mode,
"candidates": candidates,
"clicked": clicked,
},
)
else:
if mode != "ChooseWorst":
response.status = 400
return {"reason": "`mode` should be 'ChooseWorst'."}
report_history(study_id, storage, history)
report_history(
study_id,
storage,
{
"mode": mode,
"candidates": candidates,
"clicked": clicked,
},
)
response.status = 204
return {}
optuna_dashboard/_preferential_history.py
+43 -49
View File
@@ -10,7 +10,6 @@ import uuid
from optuna.storages import BaseStorage
from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE
from .preferential._system_attrs import report_preferences
@@ -19,8 +18,8 @@ _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history"
if TYPE_CHECKING:
from typing import TypedDict
NewChooseWorstHistoryJSON = TypedDict(
"NewChooseWorstHistoryJSON",
NewHistoryJSON = TypedDict(
"NewHistoryJSON",
{
"mode": Literal["ChooseWorst"],
"candidates": list[int],
@@ -28,8 +27,6 @@ if TYPE_CHECKING:
},
)
NewHistoryJSON = NewChooseWorstHistoryJSON
@dataclass(frozen=True)
class ChooseWorstHistory:
@@ -50,45 +47,6 @@ class ChooseWorstHistory:
"clicked": self.clicked,
}
def to_response(self, system_attrs: dict[str, Any]) -> dict[str, Any]:
return {
"uuid": self.uuid,
"preferences": system_attrs.get(
_SYSTEM_ATTR_PREFIX_PREFERENCE + self.preference_uuid, []
),
"timestamp": self.timestamp.isoformat(),
"candidates": self.candidates,
"clicked": self.clicked,
}
@classmethod
def from_dict(cls, d: dict[str, Any]) -> ChooseWorstHistory:
return ChooseWorstHistory(
mode="ChooseWorst",
uuid=d["uuid"],
preference_uuid=d["preference_uuid"],
timestamp=datetime.fromisoformat(d["timestamp"]),
candidates=d["candidates"],
clicked=d["clicked"],
)
@classmethod
def create(cls, study_id: int, storage: BaseStorage, d: NewHistoryJSON) -> ChooseWorstHistory:
preferences = [(best, d["clicked"]) for best in d["candidates"] if best != d["clicked"]]
preference_uuid = report_preferences(
study_id=study_id,
storage=storage,
preferences=preferences,
)
return ChooseWorstHistory(
mode="ChooseWorst",
uuid=str(uuid.uuid4()),
preference_uuid=preference_uuid,
timestamp=datetime.now(),
candidates=d["candidates"],
clicked=d["clicked"],
)
History = ChooseWorstHistory
@@ -96,9 +54,36 @@ History = ChooseWorstHistory
def report_history(
study_id: int,
storage: BaseStorage,
history: History,
input_data: NewHistoryJSON,
) -> None:
key = _SYSTEM_ATTR_PREFIX_HISTORY + history.uuid
preferences = []
if input_data["mode"] == "ChooseWorst":
preferences = [
(best, input_data["clicked"])
for best in input_data["candidates"]
if best != input_data["clicked"]
]
else:
assert False, f"Unknown mode: {input_data['mode']}"
preference_uuid = report_preferences(
study_id=study_id,
storage=storage,
preferences=preferences,
)
history_uuid = str(uuid.uuid4())
if input_data["mode"] == "ChooseWorst":
history = ChooseWorstHistory(
mode="ChooseWorst",
uuid=history_uuid,
preference_uuid=preference_uuid,
timestamp=datetime.now(),
candidates=input_data["candidates"],
clicked=input_data["clicked"],
)
key = _SYSTEM_ATTR_PREFIX_HISTORY + history_uuid
storage.set_study_system_attr(
study_id=study_id,
key=key,
@@ -115,9 +100,18 @@ def serialize_preference_history(
continue
choice: dict[str, Any] = json.loads(v)
if choice["mode"] == "ChooseWorst":
histories.append(ChooseWorstHistory.from_dict(choice))
histories.append(
ChooseWorstHistory(
mode="ChooseWorst",
uuid=choice["uuid"],
preference_uuid=choice["preference_uuid"],
timestamp=datetime.fromisoformat(choice["timestamp"]),
candidates=choice["candidates"],
clicked=choice["clicked"],
)
)
else:
assert False
assert False, f"Unknown mode: {choice['mode']}"
histories.sort(key=lambda c: c.timestamp)
return [history.to_response(system_attrs) for history in histories]
return [history.to_dict() for history in histories]
optuna_dashboard/ts/action.ts
+3 -3
View File
@@ -587,10 +587,10 @@ export const actionCreator = () => {
const updatePreference = (
study_id: number,
best_trials: number[],
worst_trials: number[]
candidates: number[],
clicked: number
) => {
reportPreferenceAPI(study_id, best_trials, worst_trials).catch((err) => {
reportPreferenceAPI(study_id, candidates, clicked).catch((err) => {
const reason = err.response?.data.reason
enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, {
variant: "error",
optuna_dashboard/ts/apiClient.ts
+7 -7
View File
@@ -58,7 +58,7 @@ const convertTrialResponse = (res: TrialResponse): Trial => {
interface PreferenceChoiceResponce {
uuid: string
candidates: number[]
preferences: number[][]
clicked: number
mode: PreferenceFeedbackMode
timestamp: string
}
@@ -68,8 +68,8 @@ const convertPreferenceChoice = (
): PreferenceChoice => {
return {
uuid: res.uuid,
candidate_trials: res.candidates,
preferences: res.preferences,
candidates: res.candidates,
clicked: res.clicked,
feedback_mode: res.mode,
timestamp: new Date(res.timestamp),
}
@@ -338,13 +338,13 @@ export const getParamImportances = (
export const reportPreferenceAPI = (
studyId: number,
best_trials: number[],
worst_trials: number[]
candidates: number[],
clicked: number
): Promise<void> => {
return axiosInstance
.post<void>(`/api/studies/${studyId}/preference`, {
candidates: best_trials.concat(worst_trials),
clicked: worst_trials[0],
candidates: candidates,
clicked: clicked,
mode: "ChooseWorst",
})
.then(() => {
optuna_dashboard/ts/components/PreferenceHistory.tsx
+2 -8
View File
@@ -138,13 +138,7 @@ const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({
trials,
}) => {
const theme = useTheme()
const worst_trials = new Set(choice.preferences.map((pair) => pair[1]))
console.log(
worst_trials,
choice.candidate_trials,
choice.preferences,
choice.preferences.map((pair) => pair[1])
)
const worst_trials = new Set([choice.clicked])
return (
<Box>
@@ -163,7 +157,7 @@ const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({
flexDirection: "row",
}}
>
{choice.candidate_trials.map((trial_num, index) => (
{choice.candidates.map((trial_num, index) => (
<CandidateTrial
key={index}
trial={trials[trial_num]}
optuna_dashboard/ts/components/PreferentialTrials.tsx
+4 -7
View File
@@ -21,9 +21,9 @@ import { MarkdownRenderer } from "./Note"
const PreferentialTrial: FC<{
trial?: Trial
studyDetail: StudyDetail
candidates: number[]
hideTrial: () => void
}> = ({ trial, studyDetail, hideTrial }) => {
}> = ({ trial, candidates, hideTrial }) => {
const theme = useTheme()
const action = actionCreator()
const trialWidth = 500
@@ -80,10 +80,7 @@ const PreferentialTrial: FC<{
aria-label="trial-button"
onClick={() => {
hideTrial()
const best_trials = studyDetail.best_trials
.map((t) => t.number)
.filter((t) => t !== trial.number)
action.updatePreference(trial.study_id, best_trials, [trial.number])
action.updatePreference(trial.study_id, candidates, trial.number)
}}
sx={{
padding: 0,
@@ -243,7 +240,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({
<PreferentialTrial
key={index}
trial={studyDetail.best_trials.find((trial) => trial.number === t)}
studyDetail={studyDetail}
candidates={displayTrials.numbers.filter((n) => n !== -1)}
hideTrial={() => {
hideTrial(t)
}}
optuna_dashboard/ts/types/index.d.ts
Vendored
+2 -2
View File
@@ -211,8 +211,8 @@ type StudyParamImportance = {
type PreferenceChoice = {
uuid: string
candidate_trials: number[]
preferences: number[][]
candidates: number[]
clicked: number
feedback_mode: PreferenceFeedbackMode
timestamp: Date
}
python_tests/test_api.py
+3 -2
View File
@@ -138,8 +138,9 @@ class APITestCase(TestCase):
"POST",
body=json.dumps(
{
"candidate_trials": [0, 1, 2],
"preferentials": [[0, 1], [2, 1]],
"mode": "ChooseWorst",
"candidates": [0, 1, 2],
"clicked": 1,
}
),
content_type="application/json",
python_tests/test_preferential_history.py
+35 -27
View File
@@ -3,7 +3,8 @@ from __future__ import annotations
from datetime import datetime
from typing import Callable
from optuna_dashboard._preferential_history import report_choice
from optuna_dashboard._preferential_history import report_history
from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE
from optuna_dashboard._serializer import serialize_preference_history
from optuna_dashboard.preferential import create_study
@@ -21,34 +22,41 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier])
study.mark_comparison_ready(trial)
study_id = study._study._study_id
report_choice(
study_id=study_id,
storage=storage,
candidate_trials=[0, 2, 3, 4],
preferences=[(2, 0), (3, 0), (4, 0)],
timestamp=datetime(2020, 1, 1, 10, 0, 1),
)
report_choice(
study_id=study_id,
storage=storage,
candidate_trials=[0, 1, 2],
preferences=[(0, 1), (2, 1)],
timestamp=datetime(2020, 1, 1, 10, 0, 0),
)
report_history(
study_id=study_id,
storage=storage,
input_data={
"mode": "ChooseWorst",
"candidates": [0, 1, 2],
"clicked": 1,
},
)
report_history(
study_id=study_id,
storage=storage,
input_data={
"mode": "ChooseWorst",
"candidates": [0, 2, 3, 4],
"clicked": 0,
},
)
history = serialize_preference_history(storage.get_study_system_attrs(study_id))
sys_attrs = storage.get_study_system_attrs(study_id)
assert len(history) == 2
assert history[0]["candidate_trials"] == [0, 1, 2]
assert len(history[0]["preferences"]) == 2
assert history[0]["candidates"] == [0, 1, 2]
assert history[0]["clicked"] == 1
preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_uuid"]]
assert len(preferences) == 2
for i, (best, worst) in enumerate([(0, 1), (2, 1)]):
assert len(history[0]["preferences"][i]) == 2
assert history[0]["preferences"][i][0] == best
assert history[0]["preferences"][i][1] == worst
assert history[0]["timestamp"] == "2020-01-01T10:00:00"
assert history[1]["candidate_trials"] == [0, 2, 3, 4]
assert len(history[1]["preferences"]) == 3
assert len(preferences[i]) == 2
assert preferences[i][0] == best
assert preferences[i][1] == worst
assert history[1]["candidates"] == [0, 2, 3, 4]
assert history[1]["clicked"] == 0
preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_uuid"]]
assert len(preferences) == 3
for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]):
assert len(history[1]["preferences"][i]) == 2
assert history[1]["preferences"][i][0] == best
assert history[1]["preferences"][i][1] == worst
assert history[1]["timestamp"] == "2020-01-01T10:00:01"
assert len(preferences[i]) == 2
assert preferences[i][0] == best
assert preferences[i][1] == worst