From a5e102d9fa250a2f431981ddc7b5f2ae39b811cf Mon Sep 17 00:00:00 2001 From: Naoto Mizuno Date: Thu, 5 Oct 2023 16:40:32 +0900 Subject: [PATCH 1/2] Make PreferentialGPSampler the default for preferential optimization --- optuna_dashboard/preferential/_study.py | 38 +++++++++++++++---------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index 6e093683..a14ef364 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -7,9 +7,9 @@ from typing import Iterable import optuna from optuna import logging +from optuna._imports import try_import from optuna.distributions import BaseDistribution from optuna.samplers import BaseSampler -from optuna.samplers import RandomSampler from optuna.trial import FrozenTrial from optuna.trial import TrialState from optuna_dashboard.preferential._system_attrs import get_n_generate @@ -20,6 +20,10 @@ from optuna_dashboard.preferential._system_attrs import report_preferences from optuna_dashboard.preferential._system_attrs import set_n_generate +with try_import() as _imports: + from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler + + _logger = logging.get_logger(__name__) _SYSTEM_ATTR_PREFERENTIAL_STUDY = "preference:is_preferential" @@ -340,11 +344,10 @@ def create_study( sampler: A sampler object that implements background algorithm for value suggestion. - If :obj:`None` is specified, `RandomSampler`_ is used. Please note that - most Optuna samplers does not work efficiently for preferential optimization. - - .. _RandomSampler: https://optuna.readthedocs.io/en/stable/reference/\ - samplers/generated/optuna.samplers.RandomSampler.html + If :obj:`None` is specified, + :class:`~optuna_dashboard.preferential.samplers.gp.PreferentialGPSampler` is used. + Please note that most Optuna samplers does not work efficiently for preferential + optimization. study_name: Study's name. If this argument is set to None, a unique name is generated @@ -361,9 +364,13 @@ def create_study( A :class:`~optuna_dashboard.preferential.PreferentialStudy` object. """ try: + if sampler is None: + _imports.check() # If BoTorch is not installed, raise ImportError. + sampler = PreferentialGPSampler() + study = optuna.create_study( storage=storage, - sampler=sampler or RandomSampler(), + sampler=sampler, study_name=study_name, ) study._storage.set_study_system_attr( @@ -433,18 +440,19 @@ def load_study( :func:`~optuna.study.create_study` for further details. sampler: A sampler object that implements background algorithm for value suggestion. - If :obj:`None` is specified, `RandomSampler`_ is used. Please note that - most Optuna samplers does not work efficiently for preferential optimization. - - .. _RandomSampler: https://optuna.readthedocs.io/en/stable/reference/samplers/\ - generated/optuna.samplers.RandomSampler.html + If :obj:`None` is specified, + :class:`~optuna_dashboard.preferential.samplers.gp.PreferentialGPSampler` is used. + Please note that most Optuna samplers does not work efficiently for preferential + optimization. Returns: A :class:`~optuna_dashboard.preferential.PreferentialStudy` object. """ - study = optuna.load_study( - study_name=study_name, storage=storage, sampler=sampler or RandomSampler() - ) + if sampler is None: + _imports.check() # If BoTorch is not installed, raise ImportError. + sampler = PreferentialGPSampler() + + study = optuna.load_study(study_name=study_name, storage=storage, sampler=sampler) system_attrs = study._storage.get_study_system_attrs(study._study_id) if not system_attrs.get(_SYSTEM_ATTR_PREFERENTIAL_STUDY): raise ValueError("The study is not a PreferentialStudy.") From 07d2bf7946dfe10d572abf38533ac52a9feeeeda Mon Sep 17 00:00:00 2001 From: Naoto Mizuno Date: Fri, 6 Oct 2023 17:54:49 +0900 Subject: [PATCH 2/2] Skip test with Python 3.7 --- python_tests/preferential/test_study.py | 5 +++++ python_tests/test_api.py | 9 +++++++++ python_tests/test_preferential_history.py | 6 ++++++ python_tests/test_serializers.py | 5 +++++ 4 files changed, 25 insertions(+) diff --git a/python_tests/preferential/test_study.py b/python_tests/preferential/test_study.py index 7fc1aaba..9b4a3f31 100644 --- a/python_tests/preferential/test_study.py +++ b/python_tests/preferential/test_study.py @@ -3,6 +3,7 @@ from __future__ import annotations import copy import multiprocessing import pickle +import sys from typing import Callable from unittest.mock import patch import uuid @@ -22,6 +23,10 @@ from ..storage_supplier import parametrize_storages from ..storage_supplier import StorageSupplier +if sys.version_info < (3, 8): + pytest.skip("BoTorch dropped Python3.7 support", allow_module_level=True) + + @parametrize_storages def test_study_set_and_get_user_attrs(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: diff --git a/python_tests/test_api.py b/python_tests/test_api.py index f9c56e4f..bde7838a 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sys from unittest import TestCase import optuna @@ -14,6 +15,7 @@ from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study +import pytest from .wsgi_client import send_request @@ -105,6 +107,7 @@ class APITestCase(TestCase): ) self.assertEqual(status, 400) + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_get_best_trials_of_preferential_study(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) @@ -128,6 +131,7 @@ class APITestCase(TestCase): assert len(best_trials) == 1 assert best_trials[0]["number"] == 0 + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_report_preference(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) @@ -161,6 +165,7 @@ class APITestCase(TestCase): assert better.number == 2 assert worse.number == 1 + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_report_preference_when_typo_mode(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) @@ -184,6 +189,7 @@ class APITestCase(TestCase): ) self.assertEqual(status, 400) + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) @@ -214,6 +220,7 @@ class APITestCase(TestCase): assert study_detail["feedback_component_type"]["output_type"] == "artifact" assert study_detail["feedback_component_type"]["artifact_key"] == "image" + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) @@ -238,6 +245,7 @@ class APITestCase(TestCase): assert len(best_trials) == 1 assert best_trials[0].number == 2 + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_remove_history(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) @@ -271,6 +279,7 @@ class APITestCase(TestCase): assert histories[0]["is_removed"] assert len(study.get_preferences()) == 0 + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_restore_history(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 3d1a7d70..dcae4797 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sys from typing import Callable from typing import TYPE_CHECKING @@ -13,6 +14,7 @@ from optuna_dashboard._preferential_history import restore_history from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE +import pytest from .storage_supplier import parametrize_storages from .storage_supplier import StorageSupplier @@ -22,6 +24,10 @@ if TYPE_CHECKING: from optuna_dashboard._preferential_history import History +if sys.version_info < (3, 8): + pytest.skip("BoTorch dropped Python3.7 support", allow_module_level=True) + + @parametrize_storages def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index ea1c3517..0d1546d2 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -1,11 +1,14 @@ from __future__ import annotations +import sys + import optuna from optuna_dashboard._serializer import serialize_attrs from optuna_dashboard._serializer import serialize_study_detail from optuna_dashboard._serializer import serialize_study_summary from optuna_dashboard._storage import get_study_summaries from optuna_dashboard.preferential import create_study +import pytest def test_serialize_bytes() -> None: @@ -22,6 +25,7 @@ def test_serialize_dict() -> None: assert len(serialized) <= 1 +@pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_get_study_detail_is_preferential() -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) @@ -48,6 +52,7 @@ def test_get_study_detail_is_not_preferential() -> None: assert not study_detail["is_preferential"] +@pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_get_study_summary_is_preferential() -> None: storage = optuna.storages.InMemoryStorage() create_study(n_generate=4, storage=storage)