From b4a4c2dae0bdac885634836ad8ba9ee871f35ab3 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 15:28:24 +0900 Subject: [PATCH 01/12] split api part of preference feedback component --- optuna_dashboard/_app.py | 23 ++++++++++ optuna_dashboard/_preference_setting.py | 60 ++++++++++++++++++++++++ optuna_dashboard/_serializer.py | 6 +++ python_tests/test_api.py | 61 +++++++++++++++++++++++++ python_tests/test_preference_setting.py | 18 ++++++++ 5 files changed, 168 insertions(+) create mode 100644 optuna_dashboard/_preference_setting.py create mode 100644 python_tests/test_preference_setting.py diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index c32c2061..fad4b74c 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,6 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preference_setting import _register_output_component from ._preferential_history import NewHistory from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route @@ -306,6 +307,28 @@ def create_app( response.status = 204 return {} + @app.post("/api/studies//component") + @json_api_view + def post_component(study_id: int) -> dict[str, Any]: + try: + component_type = request.json.get("component_type", "") + artifact_key = request.json.get("artifact_key", None) + except ValueError: + response.status = 400 + return {"reason": "invalid request."} + if component_type not in ["Note", "Artifact"]: + response.status = 400 + return {"reason": "component_type must be either 'Note' or 'Artifact'."} + + _register_output_component( + study_id=study_id, + storage=storage, + component_type=component_type, + artifact_key=artifact_key, + ) + response.status = 204 + return {} + @app.post("/api/trials//tell") @json_api_view def tell_trial(trial_id: int) -> dict[str, Any]: diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py new file mode 100644 index 00000000..93b1a34c --- /dev/null +++ b/optuna_dashboard/_preference_setting.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from optuna.storages import BaseStorage + +from .preferential._study import PreferentialStudy + + +if TYPE_CHECKING: + from typing import Literal + + OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] + +_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE = "preference:component_type" +_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY = "preference:component_artifact_key" + + +def _register_output_component( + study_id: int, + storage: BaseStorage, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str | None = None, +) -> None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, + value=component_type, + ) + if artifact_key is not None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, + value=artifact_key, + ) + + +def register_output_component( + study: PreferentialStudy, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str = "", +) -> None: + """Register output component to the study. + + Args: + study: + The study to register the output component. + component_type: + The type of the output component. + artifact_key: + When the component_type is "Artifact", + this argument is used as the attribute key of the artifact. + Each trial displays the artifact whose id is the value of the attribute. + """ + _register_output_component( + study_id=study._study._study_id, + storage=study._study._storage, + component_type=component_type, + artifact_key=artifact_key, + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index e3f77649..511e7365 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,6 +15,8 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -162,6 +164,10 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: + serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] + if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: + serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index c551e3f2..e22d104c 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,6 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study +from optuna_dashboard._preference_setting import register_output_component from optuna_dashboard.preferential import create_study from .wsgi_client import send_request @@ -179,6 +180,66 @@ class APITestCase(TestCase): ) self.assertEqual(status, 400) + def test_change_component(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + register_output_component(study, "Note") + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Artifact", "artifact_key": "image"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Artifact" + assert study_detail["feedback_artifact_key"] == "image" + + def test_change_component_type_only(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + register_output_component(study, "Artifact", "audio") + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Note"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Note" + assert study_detail["feedback_artifact_key"] == "audio" + def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py new file mode 100644 index 00000000..033a9092 --- /dev/null +++ b/python_tests/test_preference_setting.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from unittest import TestCase + +import optuna +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard.preferential._study import PreferentialStudy + + +class FeedbackSettingTestCase(TestCase): + def test_widget_to_dict_from_dict(self) -> None: + study = PreferentialStudy(optuna.create_study()) + register_output_component(study, "Artifact", "image_key") + system_attrs = study._study.system_attrs + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, "") == "Artifact" + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, "") == "image_key" From 5fcc64cff3e44bb34fb7ab9f2296fd7a8b4ff4e4 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 16:00:46 +0900 Subject: [PATCH 02/12] fix by review --- optuna_dashboard/_app.py | 10 +++--- optuna_dashboard/_preference_setting.py | 24 ++++++-------- optuna_dashboard/_serializer.py | 10 +++--- python_tests/test_api.py | 44 ++++--------------------- python_tests/test_preference_setting.py | 14 ++++---- 5 files changed, 34 insertions(+), 68 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index fad4b74c..2ad7874e 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,7 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preference_setting import _register_output_component +from ._preference_setting import _register_preference_feedback_component_type from ._preferential_history import NewHistory from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route @@ -307,11 +307,11 @@ def create_app( response.status = 204 return {} - @app.post("/api/studies//component") + @app.put("/api/studies//preference_feedback_component_type") @json_api_view - def post_component(study_id: int) -> dict[str, Any]: + def put_component(study_id: int) -> dict[str, Any]: try: - component_type = request.json.get("component_type", "") + component_type = request.json.get("type", "") artifact_key = request.json.get("artifact_key", None) except ValueError: response.status = 400 @@ -320,7 +320,7 @@ def create_app( response.status = 400 return {"reason": "component_type must be either 'Note' or 'Artifact'."} - _register_output_component( + _register_preference_feedback_component_type( study_id=study_id, storage=storage, component_type=component_type, diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 93b1a34c..73e9b489 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -12,30 +12,26 @@ if TYPE_CHECKING: OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] -_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE = "preference:component_type" -_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY = "preference:component_artifact_key" +_SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" -def _register_output_component( +def _register_preference_feedback_component_type( study_id: int, storage: BaseStorage, component_type: OUTPUT_COMPONENT_TYPE, - artifact_key: str | None = None, + artifact_key: str = "", ) -> None: storage.set_study_system_attr( study_id=study_id, - key=_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, - value=component_type, + key=_SYSTEM_ATTR_FEEDBACK_COMPONENT, + value={ + "type": component_type, + "artifact_key": artifact_key, + } ) - if artifact_key is not None: - storage.set_study_system_attr( - study_id=study_id, - key=_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, - value=artifact_key, - ) -def register_output_component( +def register_preference_feedback_component_type( study: PreferentialStudy, component_type: OUTPUT_COMPONENT_TYPE, artifact_key: str = "", @@ -52,7 +48,7 @@ def register_output_component( this argument is used as the attribute key of the artifact. Each trial displays the artifact whose id is the value of the attribute. """ - _register_output_component( + _register_preference_feedback_component_type( study_id=study._study._study_id, storage=study._study._storage, component_type=component_type, diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 511e7365..98db5ffb 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,8 +15,7 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names -from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY -from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -164,10 +163,9 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets - if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: - serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] - if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: - serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] + if serialized["is_preferential"]: + serialized["feedback_component_type"] = system_attrs.get( + _SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index e22d104c..1908c188 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,7 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard._preference_setting import register_preference_feedback_component_type from optuna_dashboard.preferential import create_study from .wsgi_client import send_request @@ -183,7 +183,7 @@ class APITestCase(TestCase): def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) - register_output_component(study, "Note") + register_preference_feedback_component_type(study, "Note") for _ in range(3): study.ask() @@ -191,9 +191,9 @@ class APITestCase(TestCase): study_id = study._study._study_id status, _, _ = send_request( app, - f"/api/studies/{study_id}/component", - "POST", - body=json.dumps({"component_type": "Artifact", "artifact_key": "image"}), + f"/api/studies/{study_id}/preference_feedback_component_type", + "PUT", + body=json.dumps({"type": "Artifact", "artifact_key": "image"}), content_type="application/json", ) self.assertEqual(status, 204) @@ -207,38 +207,8 @@ class APITestCase(TestCase): self.assertEqual(status, 200) study_detail = json.loads(body) - assert study_detail["feedback_component_type"] == "Artifact" - assert study_detail["feedback_artifact_key"] == "image" - - def test_change_component_type_only(self) -> None: - storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage, n_generate=3) - register_output_component(study, "Artifact", "audio") - for _ in range(3): - study.ask() - - app = create_app(storage) - study_id = study._study._study_id - status, _, _ = send_request( - app, - f"/api/studies/{study_id}/component", - "POST", - body=json.dumps({"component_type": "Note"}), - content_type="application/json", - ) - self.assertEqual(status, 204) - - status, _, body = send_request( - app, - f"/api/studies/{study_id}", - "GET", - content_type="application/json", - ) - self.assertEqual(status, 200) - - study_detail = json.loads(body) - assert study_detail["feedback_component_type"] == "Note" - assert study_detail["feedback_artifact_key"] == "audio" + assert study_detail["feedback_component_type"]["type"] == "Artifact" + assert study_detail["feedback_component_type"]["artifact_key"] == "image" def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index 033a9092..8a601af7 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -3,16 +3,18 @@ from __future__ import annotations from unittest import TestCase import optuna -from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY -from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE -from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT +from optuna_dashboard._preference_setting import register_preference_feedback_component_type from optuna_dashboard.preferential._study import PreferentialStudy class FeedbackSettingTestCase(TestCase): def test_widget_to_dict_from_dict(self) -> None: study = PreferentialStudy(optuna.create_study()) - register_output_component(study, "Artifact", "image_key") + register_preference_feedback_component_type(study, "Artifact", "image_key") system_attrs = study._study.system_attrs - assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, "") == "Artifact" - assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, "") == "image_key" + feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) + assert "type" in feedback_type + assert feedback_type["type"] == "Artifact" + assert "artifact_key" in feedback_type + assert feedback_type["artifact_key"] == "image_key" From fffa586da3f197b8ab3f39243d428a75c0143075 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 16:04:40 +0900 Subject: [PATCH 03/12] fix by review --- optuna_dashboard/_preference_setting.py | 2 +- optuna_dashboard/_serializer.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 73e9b489..d9d7c1c2 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -27,7 +27,7 @@ def _register_preference_feedback_component_type( value={ "type": component_type, "artifact_key": artifact_key, - } + }, ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 98db5ffb..f9aeabec 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -165,7 +165,8 @@ def serialize_study_detail( serialized["form_widgets"] = form_widgets if serialized["is_preferential"]: serialized["feedback_component_type"] = system_attrs.get( - _SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) + _SYSTEM_ATTR_FEEDBACK_COMPONENT, {} + ) if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) From 702b35622e558b00d2ba42b546d8215d750553fc Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 17:54:42 +0900 Subject: [PATCH 04/12] fix by review --- optuna_dashboard/_app.py | 2 +- optuna_dashboard/_preference_setting.py | 20 +++++++++++++------- python_tests/test_api.py | 6 +++--- python_tests/test_preference_setting.py | 4 ++-- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 2ad7874e..d7ddf72b 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -316,7 +316,7 @@ def create_app( except ValueError: response.status = 400 return {"reason": "invalid request."} - if component_type not in ["Note", "Artifact"]: + if component_type not in ["note", "artifact"]: response.status = 400 return {"reason": "component_type must be either 'Note' or 'Artifact'."} diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index d9d7c1c2..2310aa03 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING +from typing import Any from optuna.storages import BaseStorage @@ -10,7 +11,7 @@ from .preferential._study import PreferentialStudy if TYPE_CHECKING: from typing import Literal - OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] + OUTPUT_COMPONENT_TYPE = Literal["note", "artifact"] _SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" @@ -19,22 +20,22 @@ def _register_preference_feedback_component_type( study_id: int, storage: BaseStorage, component_type: OUTPUT_COMPONENT_TYPE, - artifact_key: str = "", + artifact_key: str | None = None, ) -> None: + value: dict[str, Any] = {"type": component_type} + if artifact_key is not None: + value["artifact_key"] = artifact_key storage.set_study_system_attr( study_id=study_id, key=_SYSTEM_ATTR_FEEDBACK_COMPONENT, - value={ - "type": component_type, - "artifact_key": artifact_key, - }, + value=value, ) def register_preference_feedback_component_type( study: PreferentialStudy, component_type: OUTPUT_COMPONENT_TYPE, - artifact_key: str = "", + artifact_key: str | None = None, ) -> None: """Register output component to the study. @@ -48,6 +49,11 @@ def register_preference_feedback_component_type( this argument is used as the attribute key of the artifact. Each trial displays the artifact whose id is the value of the attribute. """ + if component_type == "artifact": + assert ( + artifact_key is not None + ), "artifact_key must be specified when component_type is Artifact" + _register_preference_feedback_component_type( study_id=study._study._study_id, storage=study._study._storage, diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 1908c188..c478dc26 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -183,7 +183,7 @@ class APITestCase(TestCase): def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) - register_preference_feedback_component_type(study, "Note") + register_preference_feedback_component_type(study, "note") for _ in range(3): study.ask() @@ -193,7 +193,7 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference_feedback_component_type", "PUT", - body=json.dumps({"type": "Artifact", "artifact_key": "image"}), + body=json.dumps({"type": "artifact", "artifact_key": "image"}), content_type="application/json", ) self.assertEqual(status, 204) @@ -207,7 +207,7 @@ class APITestCase(TestCase): self.assertEqual(status, 200) study_detail = json.loads(body) - assert study_detail["feedback_component_type"]["type"] == "Artifact" + assert study_detail["feedback_component_type"]["type"] == "artifact" assert study_detail["feedback_component_type"]["artifact_key"] == "image" def test_skip_trial(self) -> None: diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index 8a601af7..bcd53e42 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -11,10 +11,10 @@ from optuna_dashboard.preferential._study import PreferentialStudy class FeedbackSettingTestCase(TestCase): def test_widget_to_dict_from_dict(self) -> None: study = PreferentialStudy(optuna.create_study()) - register_preference_feedback_component_type(study, "Artifact", "image_key") + register_preference_feedback_component_type(study, "artifact", "image_key") system_attrs = study._study.system_attrs feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) assert "type" in feedback_type - assert feedback_type["type"] == "Artifact" + assert feedback_type["type"] == "artifact" assert "artifact_key" in feedback_type assert feedback_type["artifact_key"] == "image_key" From 8e23d30d182f757def64b0c907d7d88a6517e45e Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 18:18:19 +0900 Subject: [PATCH 05/12] fix by format --- optuna_dashboard/_preference_setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 2310aa03..acc64ccd 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING from typing import Any +from typing import TYPE_CHECKING from optuna.storages import BaseStorage From 32a4f2d4e1c267dba695c9e4828c376e32e5eb95 Mon Sep 17 00:00:00 2001 From: contramundum53 Date: Thu, 14 Sep 2023 10:36:20 +0900 Subject: [PATCH 06/12] Update test_api.py --- python_tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 2e46be43..d5de330d 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,8 +8,8 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preference_setting import register_preference_feedback_component_type +from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history from optuna_dashboard._serializer import serialize_preference_history From 88e3d511d5ff607d44b780205849b92efa6b6792 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 12:03:17 +0900 Subject: [PATCH 07/12] fix by review --- optuna_dashboard/_app.py | 4 +++- optuna_dashboard/_serializer.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 1baf76fe..4c22bdbf 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -312,7 +312,7 @@ def create_app( @app.put("/api/studies//preference_feedback_component_type") @json_api_view - def put_component(study_id: int) -> dict[str, Any]: + def put_preference_feedback_component_type(study_id: int) -> dict[str, Any]: try: component_type = request.json.get("type", "") artifact_key = request.json.get("artifact_key", None) @@ -329,6 +329,8 @@ def create_app( component_type=component_type, artifact_key=artifact_key, ) + response.status = 204 + return {} @app.delete("/api/studies//preference/") @json_api_view diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index ea22644f..b229b738 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -168,7 +168,6 @@ def serialize_study_detail( serialized["feedback_component_type"] = system_attrs.get( _SYSTEM_ATTR_FEEDBACK_COMPONENT, {} ) - if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) serialized["plotly_graph_objects"] = [ From e5e6d0766f69c69332d3c9d149e71f8638ade1c1 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 15:37:48 +0900 Subject: [PATCH 08/12] fix docstring --- optuna_dashboard/_preference_setting.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index acc64ccd..167fd178 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -37,17 +37,22 @@ def register_preference_feedback_component_type( component_type: OUTPUT_COMPONENT_TYPE, artifact_key: str | None = None, ) -> None: - """Register output component to the study. + """Register a preference feedback component to the study. + With this feature, you can change the component, displayed on the + human feedback pages. By default, the Markdown note (``component_type="note"``) + is displayed. If you specify ``component_type="artifact"``, the viewer for the + specified artifact file will be displayed. Args: study: - The study to register the output component. + The study to register the preference feedback component. component_type: - The type of the output component. - artifact_key: - When the component_type is "Artifact", - this argument is used as the attribute key of the artifact. - Each trial displays the artifact whose id is the value of the attribute. + The component type, displayed on the human feedback pages + (default: ``"note"``). + user_attr_artifact_key: + This option is required when the ``component_type`` is ``"artifact"``. + The user attribute, which is specified this field, must contain the + ``artifact``id you want to display on the human feedback page. """ if component_type == "artifact": assert ( From 9d7f2f9a6e141450f388747782ef9de2f228b141 Mon Sep 17 00:00:00 2001 From: moririn2528 <49509238+moririn2528@users.noreply.github.com> Date: Thu, 14 Sep 2023 16:28:43 +0900 Subject: [PATCH 09/12] Update optuna_dashboard/_preference_setting.py Co-authored-by: c-bata --- optuna_dashboard/_preference_setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 167fd178..d0f8c82c 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -32,7 +32,7 @@ def _register_preference_feedback_component_type( ) -def register_preference_feedback_component_type( +def register_preference_feedback_component( study: PreferentialStudy, component_type: OUTPUT_COMPONENT_TYPE, artifact_key: str | None = None, From 4570efe74db693276482aa946bfec4be7cec3064 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 16:39:44 +0900 Subject: [PATCH 10/12] fix by review --- docs/api.rst | 1 + optuna_dashboard/_app.py | 4 ++-- optuna_dashboard/_preference_setting.py | 4 ++-- python_tests/test_api.py | 4 ++-- python_tests/test_preference_setting.py | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index aadd2718..e66ae468 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,6 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy + optuna_dashboard._prefential_setting.register_preference_feedback_component Streamlit ----------------- diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 4c22bdbf..3abac3fc 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,7 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preference_setting import _register_preference_feedback_component_type +from ._preference_setting import _register_preference_feedback_component from ._preferential_history import NewHistory from ._preferential_history import PreferenceHistoryNotFound from ._preferential_history import remove_history @@ -323,7 +323,7 @@ def create_app( response.status = 400 return {"reason": "component_type must be either 'Note' or 'Artifact'."} - _register_preference_feedback_component_type( + _register_preference_feedback_component( study_id=study_id, storage=storage, component_type=component_type, diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index d0f8c82c..907fc2be 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: _SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" -def _register_preference_feedback_component_type( +def _register_preference_feedback_component( study_id: int, storage: BaseStorage, component_type: OUTPUT_COMPONENT_TYPE, @@ -59,7 +59,7 @@ def register_preference_feedback_component( artifact_key is not None ), "artifact_key must be specified when component_type is Artifact" - _register_preference_feedback_component_type( + _register_preference_feedback_component( study_id=study._study._study_id, storage=study._study._storage, component_type=component_type, diff --git a/python_tests/test_api.py b/python_tests/test_api.py index d5de330d..db79c0d8 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,7 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard._preference_setting import register_preference_feedback_component_type +from optuna_dashboard._preference_setting import register_preference_feedback_component from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history @@ -187,7 +187,7 @@ class APITestCase(TestCase): def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) - register_preference_feedback_component_type(study, "note") + register_preference_feedback_component(study, "note") for _ in range(3): study.ask() diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index bcd53e42..a0adcb39 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -4,14 +4,14 @@ from unittest import TestCase import optuna from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT -from optuna_dashboard._preference_setting import register_preference_feedback_component_type +from optuna_dashboard._preference_setting import register_preference_feedback_component from optuna_dashboard.preferential._study import PreferentialStudy class FeedbackSettingTestCase(TestCase): def test_widget_to_dict_from_dict(self) -> None: study = PreferentialStudy(optuna.create_study()) - register_preference_feedback_component_type(study, "artifact", "image_key") + register_preference_feedback_component(study, "artifact", "image_key") system_attrs = study._study.system_attrs feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) assert "type" in feedback_type From df1948a484f896aac4ef9e3576c36c7481a6ac13 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 16:42:58 +0900 Subject: [PATCH 11/12] minor fix --- docs/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index e66ae468..8da91536 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,7 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy - optuna_dashboard._prefential_setting.register_preference_feedback_component + optuna_dashboard._preference_setting.register_preference_feedback_component Streamlit ----------------- From 9f17603741a73770ebebec2d87356d2d1ef7ccda Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 17:05:12 +0900 Subject: [PATCH 12/12] minor fix --- docs/api.rst | 2 +- optuna_dashboard/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index 8da91536..18f8bc72 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,7 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy - optuna_dashboard._preference_setting.register_preference_feedback_component + optuna_dashboard.register_preference_feedback_component Streamlit ----------------- diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 5bb3f301..ea2a8dbe 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -14,6 +14,7 @@ from ._form_widget import TextInputWidget # noqa from ._named_objectives import set_objective_names # noqa from ._note import get_note # noqa from ._note import save_note # noqa +from ._preference_setting import register_preference_feedback_component # noqa __version__ = "0.13.0b1"