From 9564731ce97a5ee2a94ca9b18ecb83cf06ce0d9c Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 29 Aug 2023 16:58:51 +0900 Subject: [PATCH] add test and fix by lint --- optuna_dashboard/_app.py | 6 +-- optuna_dashboard/_serializer.py | 6 ++- optuna_dashboard/preferential/_history.py | 28 +++++----- .../ts/components/PreferenceHistory.tsx | 2 +- python_tests/preferential/test_history.py | 51 +++++++++++++++++++ python_tests/test_api.py | 8 ++- 6 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 python_tests/preferential/test_history.py diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index e18f37fc..bf32cb23 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import functools import logging import os @@ -8,7 +9,6 @@ from typing import Any from typing import Optional from typing import Union import warnings -from datetime import datetime from bottle import Bottle from bottle import redirect @@ -39,10 +39,10 @@ from ._storage_url import get_storage from .artifact._backend import delete_all_artifacts from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store -from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY -from .preferential._study import get_best_trials as get_best_preferential_trials from .preferential._history import FeedbackMode from .preferential._history import report_choice +from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY +from .preferential._study import get_best_trials as get_best_preferential_trials if typing.TYPE_CHECKING: diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 372cf458..741385ca 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,10 +15,11 @@ from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names from .artifact._backend import list_trial_artifacts +from .preferential._history import _SYSTEM_ATTR_PREFIX_HISTORY from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY -from .preferential._history import Choice, _SYSTEM_ATTR_PREFIX_HISTORY from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE + if TYPE_CHECKING: from typing import Literal from typing import TypedDict @@ -334,7 +335,7 @@ def serialize_search_space( def serialize_preference_history( system_attrs: dict[str, Any], ) -> list[dict[str, Any]]: - history: list[Choice] = [] + history: list[dict[str, Any]] = [] for k, v in system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): continue @@ -343,4 +344,5 @@ def serialize_preference_history( _SYSTEM_ATTR_PREFIX_PREFERENCE + choice["preference_uuid"], [] ) history.append(choice) + history.sort(key=lambda c: c["timestamp"]) return history diff --git a/optuna_dashboard/preferential/_history.py b/optuna_dashboard/preferential/_history.py index 15a3ceb2..4170ee21 100644 --- a/optuna_dashboard/preferential/_history.py +++ b/optuna_dashboard/preferential/_history.py @@ -1,15 +1,15 @@ -from enum import Enum +from dataclasses import asdict +from dataclasses import dataclass from datetime import datetime -import uuid +from enum import Enum import json -from dataclasses import dataclass, asdict -from typing import Any from json import JSONEncoder +from typing import Any +import uuid from optuna.storages import BaseStorage from ._system_attrs import report_preferences -from .._storage import get_study_summary _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" @@ -30,14 +30,14 @@ class Choice: class Encoder(JSONEncoder): - def default(self, o): - if isinstance(o, FeedbackMode): - return o.name - if isinstance(o, Choice): - return asdict(o) - if isinstance(o, datetime): - return o.isoformat() - return super().default(o) + def default(self, a: Any) -> Any: + if isinstance(a, FeedbackMode): + return a.name + if isinstance(a, Choice): + return asdict(a) + if isinstance(a, datetime): + return a.isoformat() + return super().default(a) def report_choice( @@ -47,7 +47,7 @@ def report_choice( preferences: list[tuple[int, int]], feedback_mode: FeedbackMode, timestamp: datetime, -): +) -> None: choice = Choice( uuid=str(uuid.uuid4()), candidate_trials=candidate_trials, diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 27e3b1b7..1e4c9346 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -27,7 +27,7 @@ const CandidateTrial: FC<{ const trialHeight = 300 const [detailShown, setDetailShown] = useState(false) - let cardComponentSx = { + const cardComponentSx = { padding: 0, position: "relative", overflow: "hidden", diff --git a/python_tests/preferential/test_history.py b/python_tests/preferential/test_history.py new file mode 100644 index 00000000..e8563b25 --- /dev/null +++ b/python_tests/preferential/test_history.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Callable + +from optuna_dashboard._serializer import serialize_preference_history +from optuna_dashboard.preferential import create_study +from optuna_dashboard.preferential._history import FeedbackMode +from optuna_dashboard.preferential._history import report_choice + +from ..storage_supplier import parametrize_storages +from ..storage_supplier import StorageSupplier + + +@parametrize_storages +def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) + study.mark_comparison_ready(trial) + + study_id = study._study._study_id + report_choice( + study_id=study_id, + storage=storage, + candidate_trials=[0, 2, 3, 4], + preferences=[(2, 0), (3, 0), (4, 0)], + feedback_mode=FeedbackMode.CHOOSE_WORST, + timestamp=datetime(2020, 1, 1, 10, 0, 1), + ) + report_choice( + study_id=study_id, + storage=storage, + candidate_trials=[0, 1, 2], + preferences=[(0, 1), (2, 1)], + feedback_mode=FeedbackMode.CHOOSE_WORST, + timestamp=datetime(2020, 1, 1, 10, 0, 0), + ) + + history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + assert len(history) == 2 + assert history[0]["candidate_trials"] == [0, 1, 2] + assert history[0]["preferences"] == [[0, 1], [2, 1]] + assert history[0]["feedback_mode"] == FeedbackMode.CHOOSE_WORST.name + assert history[0]["timestamp"] == "2020-01-01T10:00:00" + assert history[1]["candidate_trials"] == [0, 2, 3, 4] + assert history[1]["preferences"] == [[2, 0], [3, 0], [4, 0]] + assert history[1]["feedback_mode"] == FeedbackMode.CHOOSE_WORST.name + assert history[1]["timestamp"] == "2020-01-01T10:00:01" diff --git a/python_tests/test_api.py b/python_tests/test_api.py index c995c0e2..1909b461 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -136,7 +136,13 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference", "POST", - body=json.dumps({"best_trials": [0, 2], "worst_trials": [1]}), + body=json.dumps( + { + "candidate_trials": [0, 1, 2], + "preferentials": [[0, 1], [2, 1]], + "mode": "choose_worst", + } + ), content_type="application/json", ) self.assertEqual(status, 204)