Merge branch 'main' into support-optuna-study-artifacts

python_tests/preferential/test_samplers.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+import sys
+
+import optuna
+from optuna import create_trial
+from optuna.distributions import CategoricalDistribution
+from optuna.distributions import FloatDistribution
+from optuna.distributions import IntDistribution
+from optuna.samplers import BaseSampler
+from optuna.trial import TrialState
+from optuna_dashboard.preferential import create_study
+import pytest
+
+
+if sys.version_info >= (3, 8):
+    from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler
+else:
+    PreferentialGPSampler = None
+
+
+parametrize_sampler = pytest.mark.parametrize(
+    "sampler_class",
+    [
+        optuna.samplers.RandomSampler,
+        pytest.param(
+            PreferentialGPSampler,
+            marks=pytest.mark.skipif(
+                sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support"
+            ),
+        ),
+    ],
+)
+
+
+@parametrize_sampler
+def test_sample_float(sampler_class: Callable[[], BaseSampler]) -> None:
+    study = create_study(n_generate=4, sampler=sampler_class())
+
+    for i in range(5):
+        past_trial = create_trial(
+            state=TrialState.RUNNING,
+            params={"x": 1.0},
+            distributions={"x": FloatDistribution(0, 10)},
+        )
+        study.add_trial(past_trial)
+    study.report_preference(study.trials[:-1], study.trials[-1])
+
+    trial = study.ask()
+    trial.suggest_float("x", 0, 10)
+
+
+@parametrize_sampler
+def test_sample_int(sampler_class: Callable[[], BaseSampler]) -> None:
+    study = create_study(n_generate=4, sampler=sampler_class())
+
+    for i in range(5):
+        past_trial = create_trial(
+            state=TrialState.RUNNING,
+            params={"x": 1},
+            distributions={"x": IntDistribution(0, 10)},
+        )
+        study.add_trial(past_trial)
+    study.report_preference(study.trials[:-1], study.trials[-1])
+
+    trial = study.ask()
+    trial.suggest_int("x", 0, 10)
+
+
+@parametrize_sampler
+def test_sample_categorical(sampler_class: Callable[[], BaseSampler]) -> None:
+    study = create_study(n_generate=4, sampler=sampler_class())
+
+    for i in range(5):
+        past_trial = create_trial(
+            state=TrialState.RUNNING,
+            params={"x": "A"},
+            distributions={"x": CategoricalDistribution(["A", "B", "C"])},
+        )
+        study.add_trial(past_trial)
+    study.report_preference(study.trials[:-1], study.trials[-1])
+
+    trial = study.ask()
+    trial.suggest_categorical("x", ["A", "B", "C"])
+
+
+@parametrize_sampler
+def test_sample_mixed(sampler_class: Callable[[], BaseSampler]) -> None:
+    study = create_study(n_generate=4, sampler=sampler_class())
+
+    for i in range(5):
+        past_trial = create_trial(
+            state=TrialState.RUNNING,
+            params={"x": 1.0, "y": 1, "z": "A"},
+            distributions={
+                "x": FloatDistribution(0, 10),
+                "y": IntDistribution(0, 10),
+                "z": CategoricalDistribution(["A", "B", "C"]),
+            },
+        )
+        study.add_trial(past_trial)
+    study.report_preference(study.trials[:-1], study.trials[-1])
+
+    trial = study.ask()
+    trial.suggest_float("x", 0, 10)
+    trial.suggest_int("y", 0, 10)
+    trial.suggest_categorical("z", ["A", "B", "C"])
+
+
+@parametrize_sampler
+def test_sample_first_trial(sampler_class: Callable[[], BaseSampler]) -> None:
+    study = create_study(n_generate=4, sampler=sampler_class())
+    trial = study.ask()
+    trial.suggest_float("x", 0, 10)
+
+
+@parametrize_sampler
+def test_sample_dynamic_search_space(sampler_class: Callable[[], BaseSampler]) -> None:
+    study = create_study(n_generate=4, sampler=sampler_class())
+
+    for i in range(5):
+        past_trial = create_trial(
+            state=TrialState.RUNNING,
+            params={"x": 1.0},
+            distributions={"x": FloatDistribution(0, 10)},
+        )
+        study.add_trial(past_trial)
+    study.report_preference(study.trials[:-1], study.trials[-1])
+
+    trial = study.ask()
+    trial.suggest_float("x", -100, 100)

python_tests/test_api.py

@@ -8,6 +8,11 @@ from optuna import get_all_study_summaries
 from optuna.study import StudyDirection
 from optuna_dashboard._app import create_app
 from optuna_dashboard._app import create_new_study
+from optuna_dashboard._preference_setting import register_preference_feedback_component
+from optuna_dashboard._preferential_history import NewHistory
+from optuna_dashboard._preferential_history import remove_history
+from optuna_dashboard._preferential_history import report_history
+from optuna_dashboard._serializer import serialize_preference_history
 from optuna_dashboard.preferential import create_study
 
 from .wsgi_client import send_request
@@ -179,6 +184,36 @@ class APITestCase(TestCase):
         )
         self.assertEqual(status, 400)
 
+    def test_change_component(self) -> None:
+        storage = optuna.storages.InMemoryStorage()
+        study = create_study(storage=storage, n_generate=3)
+        register_preference_feedback_component(study, "note")
+        for _ in range(3):
+            study.ask()
+
+        app = create_app(storage)
+        study_id = study._study._study_id
+        status, _, _ = send_request(
+            app,
+            f"/api/studies/{study_id}/preference_feedback_component",
+            "PUT",
+            body=json.dumps({"output_type": "artifact", "artifact_key": "image"}),
+            content_type="application/json",
+        )
+        self.assertEqual(status, 204)
+
+        status, _, body = send_request(
+            app,
+            f"/api/studies/{study_id}",
+            "GET",
+            content_type="application/json",
+        )
+        self.assertEqual(status, 200)
+
+        study_detail = json.loads(body)
+        assert study_detail["feedback_component_type"]["output_type"] == "artifact"
+        assert study_detail["feedback_component_type"]["artifact_key"] == "image"
+
     def test_skip_trial(self) -> None:
         storage = optuna.storages.InMemoryStorage()
         study = create_study(n_generate=4, storage=storage)
@@ -203,6 +238,82 @@ class APITestCase(TestCase):
         assert len(best_trials) == 1
         assert best_trials[0].number == 2
 
+    def test_remove_history(self) -> None:
+        storage = optuna.storages.InMemoryStorage()
+        study = create_study(storage=storage, n_generate=3)
+        for _ in range(3):
+            study.ask()
+
+        app = create_app(storage)
+        study_id = study._study._study_id
+        history_id = report_history(
+            study_id,
+            storage,
+            NewHistory(
+                mode="ChooseWorst",
+                candidates=[0, 1, 2],
+                clicked=2,
+            ),
+        )
+        histories = serialize_preference_history(storage.get_study_system_attrs(study_id))
+        assert len(histories) == 1
+        assert not histories[0]["is_removed"]
+
+        status, _, _ = send_request(
+            app,
+            f"/api/studies/{study_id}/preference/{history_id}",
+            "DELETE",
+            content_type="application/json",
+        )
+        self.assertEqual(status, 204)
+        histories = serialize_preference_history(storage.get_study_system_attrs(study_id))
+        assert len(histories) == 1
+        assert histories[0]["is_removed"]
+        assert len(study.get_preferences()) == 0
+
+    def test_restore_history(self) -> None:
+        storage = optuna.storages.InMemoryStorage()
+        study = create_study(storage=storage, n_generate=3)
+        for _ in range(3):
+            study.ask()
+
+        app = create_app(storage)
+        study_id = study._study._study_id
+        history_id = report_history(
+            study_id,
+            storage,
+            NewHistory(
+                mode="ChooseWorst",
+                candidates=[0, 1, 2],
+                clicked=2,
+            ),
+        )
+        remove_history(study_id, storage, history_id)
+        histories = serialize_preference_history(storage.get_study_system_attrs(study_id))
+        assert len(histories) == 1
+        assert histories[0]["is_removed"]
+        assert len(study.get_preferences()) == 0
+
+        status, _, _ = send_request(
+            app,
+            f"/api/studies/{study_id}/preference/{history_id}",
+            "POST",
+            content_type="application/json",
+        )
+        self.assertEqual(status, 204)
+        histories = serialize_preference_history(storage.get_study_system_attrs(study_id))
+        assert len(histories) == 1
+        assert not histories[0]["is_removed"]
+        preferences = study.get_preferences()
+        preferences.sort(key=lambda x: (x[0].number, x[1].number))
+        assert len(preferences) == 2
+        better, worse = preferences[0]
+        assert better.number == 0
+        assert worse.number == 2
+        better, worse = preferences[1]
+        assert better.number == 1
+        assert worse.number == 2
+
     def test_create_study(self) -> None:
         for name, directions, expected_status in [
             ("single-objective success", ["minimize"], 201),

python_tests/test_preference_setting.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from unittest import TestCase
+
+import optuna
+from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT
+from optuna_dashboard._preference_setting import register_preference_feedback_component
+from optuna_dashboard.preferential._study import PreferentialStudy
+
+
+class FeedbackSettingTestCase(TestCase):
+    def test_widget_to_dict_from_dict(self) -> None:
+        study = PreferentialStudy(optuna.create_study())
+        register_preference_feedback_component(study, "artifact", "image_key")
+        system_attrs = study._study.system_attrs
+        feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {})
+        assert "output_type" in feedback_type
+        assert feedback_type["output_type"] == "artifact"
+        assert "artifact_key" in feedback_type
+        assert feedback_type["artifact_key"] == "image_key"

python_tests/test_preferential_history.py

@@ -1,9 +1,15 @@
 from __future__ import annotations
 
+import json
 from typing import Callable
+from typing import TYPE_CHECKING
 
+from optuna.storages import BaseStorage
+from optuna_dashboard._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY
 from optuna_dashboard._preferential_history import NewHistory
+from optuna_dashboard._preferential_history import remove_history
 from optuna_dashboard._preferential_history import report_history
+from optuna_dashboard._preferential_history import restore_history
 from optuna_dashboard._serializer import serialize_preference_history
 from optuna_dashboard.preferential import create_study
 from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE
@@ -12,6 +18,10 @@ from .storage_supplier import parametrize_storages
 from .storage_supplier import StorageSupplier
 
 
+if TYPE_CHECKING:
+    from optuna_dashboard._preferential_history import History
+
+
 @parametrize_storages
 def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None:
     with storage_supplier() as storage:
@@ -25,37 +35,102 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier])
         report_history(
             study_id=study_id,
             storage=storage,
-            input_data=NewHistory(
-                mode="ChooseWorst",
-                candidates=[0, 1, 2],
-                clicked=1,
-            ),
+            input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1),
         )
         report_history(
             study_id=study_id,
             storage=storage,
-            input_data=NewHistory(
-                mode="ChooseWorst",
-                candidates=[0, 2, 3, 4],
-                clicked=0,
-            ),
+            input_data=NewHistory(mode="ChooseWorst", candidates=[0, 2, 3, 4], clicked=0),
         )
         history = serialize_preference_history(storage.get_study_system_attrs(study_id))
         sys_attrs = storage.get_study_system_attrs(study_id)
         assert len(history) == 2
-        assert history[0]["candidates"] == [0, 1, 2]
-        assert history[0]["clicked"] == 1
-        preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]]
+        assert history[0]["history"]["candidates"] == [0, 1, 2]
+        assert history[0]["history"]["clicked"] == 1
+        preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["history"]["id"]]
         assert len(preferences) == 2
         for i, (best, worst) in enumerate([(0, 1), (2, 1)]):
             assert len(preferences[i]) == 2
             assert preferences[i][0] == best
             assert preferences[i][1] == worst
-        assert history[1]["candidates"] == [0, 2, 3, 4]
-        assert history[1]["clicked"] == 0
-        preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]]
+        assert history[1]["history"]["candidates"] == [0, 2, 3, 4]
+        assert history[1]["history"]["clicked"] == 0
+        preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["history"]["id"]]
         assert len(preferences) == 3
         for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]):
             assert len(preferences[i]) == 2
             assert preferences[i][0] == best
             assert preferences[i][1] == worst
+
+
+def get_preferences_history(
+    study_id: int,
+    storage: BaseStorage,
+    history_id: str,
+) -> tuple[list[tuple[int, int]], History]:
+    system_attrs = storage.get_study_system_attrs(study_id)
+    history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + history_id, ""))
+    preference: list[tuple[int, int]] = system_attrs.get(
+        _SYSTEM_ATTR_PREFIX_PREFERENCE + history_id, []
+    )
+    return preference, history
+
+
+@parametrize_storages
+def test_remove_history(storage_supplier: Callable[[], StorageSupplier]) -> None:
+    with storage_supplier() as storage:
+        study = create_study(storage=storage, n_generate=5)
+        for _ in range(5):
+            trial = study.ask()
+            trial.suggest_float("x", 0, 1)
+        study_id = study._study._study_id
+
+        history_id = report_history(
+            study_id=study_id,
+            storage=storage,
+            input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1),
+        )
+        remove_history(study_id, storage, history_id)
+        preference, history = get_preferences_history(study_id, storage, history_id)
+        assert history["mode"] == "ChooseWorst"
+        assert history["candidates"] == [0, 1, 2]
+        assert history["clicked"] == 1
+        assert len(preference) == 0
+
+        remove_history(study_id, storage, history_id)
+        preference, history = get_preferences_history(study_id, storage, history_id)
+        assert len(preference) == 0
+
+
+@parametrize_storages
+def test_restore_history(storage_supplier: Callable[[], StorageSupplier]) -> None:
+    with storage_supplier() as storage:
+        study = create_study(storage=storage, n_generate=5)
+        for _ in range(5):
+            trial = study.ask()
+            trial.suggest_float("x", 0, 1)
+        study_id = study._study._study_id
+
+        history_id = report_history(
+            study_id=study_id,
+            storage=storage,
+            input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1),
+        )
+        remove_history(study_id, storage, history_id)
+        preference, history = get_preferences_history(study_id, storage, history_id)
+        assert len(preference) == 0
+
+        restore_history(study_id, storage, history_id)
+        preference, history = get_preferences_history(study_id, storage, history_id)
+        assert history["mode"] == "ChooseWorst"
+        assert history["candidates"] == [0, 1, 2]
+        assert history["clicked"] == 1
+        assert len(preference) == 2
+        for i, (best, worst) in enumerate([(0, 1), (2, 1)]):
+            assert len(preference[i]) == 2
+            assert preference[i][0] == best
+            assert preference[i][1] == worst
+
+        restore_history(study_id, storage, history_id)
+        preference, history = get_preferences_history(study_id, storage, history_id)
+        assert len(preference) == 2

python_tests/test_serializers.py

@@ -29,7 +29,9 @@ def test_get_study_detail_is_preferential() -> None:
     assert len(study_summaries) == 1
 
     study_summary = study_summaries[0]
-    study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {})
+    study_detail = serialize_study_detail(
+        study_summary, [], study.trials, [], [], [], False, {}, []
+    )
     assert study_detail["is_preferential"]
 
 
@@ -40,7 +42,9 @@ def test_get_study_detail_is_not_preferential() -> None:
     assert len(study_summaries) == 1
 
     study_summary = study_summaries[0]
-    study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {})
+    study_detail = serialize_study_detail(
+        study_summary, [], study.trials, [], [], [], False, {}, []
+    )
     assert not study_detail["is_preferential"]