From 4905890414aa9ecdc7a0a10e8d48601a9900adc8 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 20 Aug 2023 16:16:31 +0900 Subject: [PATCH 01/11] Devide test db in e2e tests --- Makefile | 2 +- e2e_tests/__init__.py | 0 e2e_tests/conftest.py | 245 ------------------ e2e_tests/mock_db.py | 35 +++ e2e_tests/test_usecases/__init__.py | 0 e2e_tests/test_usecases/test_study_history.py | 30 +++ e2e_tests/visual_regression_test.py | 220 ++++++++++++++++ 7 files changed, 286 insertions(+), 246 deletions(-) create mode 100644 e2e_tests/__init__.py create mode 100644 e2e_tests/mock_db.py create mode 100644 e2e_tests/test_usecases/__init__.py diff --git a/Makefile b/Makefile index f925bb78..4094fd57 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ docs: docs/conf.py $(RST_FILES) .PHONY: fmt fmt: npm run fmt - black ./optuna_dashboard/ ./python_tests/ + black ./optuna_dashboard/ ./python_tests/ ./e2e_tests/ isort . .PHONY: clean diff --git a/e2e_tests/__init__.py b/e2e_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 8d8eac3b..ad1190cb 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -1,251 +1,6 @@ -import socket -import threading -from wsgiref.simple_server import make_server - -import optuna -from optuna_dashboard import wsgi import pytest -study_names = [ - "single", - "single-trial", - "single-1-param", - "single-dynamic", - "single-inf", - "multi-objective", - "multi-dynamic", - "single-pruned-without-report", - "single-inf-report", - "issue-410", - "single-no-trials", - "multi-no-trials", -] - - -def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage: - storage = optuna.storages.InMemoryStorage() - sampler = optuna.samplers.RandomSampler(seed=0) - - # Sinble objective study - if study_name == "single": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 - - study.optimize(objective_single, n_trials=50) - - # A single objective study with a single trial - # Refs: https://github.com/optuna/optuna-dashboard/issues/401 - elif study_name == "single-trial": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 - - study.optimize(objective_single, n_trials=1) - - # Single-objective study with 1 parameter - elif study_name == "single-1-param": - study = optuna.create_study( - study_name=study_name, storage=storage, direction="maximize", sampler=sampler - ) - - def objective_single_with_1param(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - return -((x1 - 2) ** 2) - - study.optimize(objective_single_with_1param, n_trials=50) - - # Single-objective study with dynamic search space - elif study_name == "single-dynamic": - study = optuna.create_study( - study_name=study_name, storage=storage, direction="maximize", sampler=sampler - ) - - def objective_single_dynamic(trial: optuna.Trial) -> float: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - return (trial.suggest_float("x1", 0, 10) - 2) ** 2 - else: - return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) - - study.optimize(objective_single_dynamic, n_trials=50) - - # Single objective study with 'inf', '-inf', or 'nan' value - elif study_name == "single-inf": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single_inf(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - return float("inf") - elif trial.number % 3 == 1: - return float("-inf") - else: - return x**2 - - study.optimize(objective_single_inf, n_trials=50) - - # Multi-objective study - elif study_name == "multi-objective": - study = optuna.create_study( - study_name=study_name, - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) - - def objective_multi(trial: optuna.Trial) -> tuple[float, float]: - x = trial.suggest_float("x", 0, 5) - y = trial.suggest_float("y", 0, 3) - v0 = 4 * x**2 + 4 * y**2 - v1 = (x - 5) ** 2 + (y - 5) ** 2 - return v0, v1 - - study.optimize(objective_multi, n_trials=50) - - # Multi-objective study with dynamic search space - elif study_name == "multi-dynamic": - study = optuna.create_study( - study_name=study_name, - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) - - def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - x = trial.suggest_float("x1", 0, 5) - y = trial.suggest_float("y1", 0, 3) - v0 = 4 * x**2 + 4 * y**2 - v1 = (x - 5) ** 2 + (y - 5) ** 2 - return v0, v1 - else: - x = trial.suggest_float("x2", 0, 5) - y = trial.suggest_float("y2", 0, 3) - v0 = 2 * x**2 + 2 * y**2 - v1 = (x - 2) ** 2 + (y - 3) ** 2 - return v0, v1 - - study.optimize(objective_multi_dynamic, n_trials=50) - - # Pruning with no intermediate values - elif study_name == "single-pruned-without-report": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_prune_without_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -15, 30) - y = trial.suggest_float("y", -15, 30) - v = x**2 + y**2 - if v > 100: - raise optuna.TrialPruned() - return v - - study.optimize(objective_prune_without_report, n_trials=100) - - # Single objective pruned after reported 'inf', '-inf', or 'nan' - elif study_name == "single-inf-report": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single_inf_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - trial.report(float("inf"), 1) - elif trial.number % 3 == 1: - trial.report(float("-inf"), 1) - else: - trial.report(float("nan"), 1) - - if x > 0: - raise optuna.TrialPruned() - else: - return x**2 - - study.optimize(objective_single_inf_report, n_trials=50) - - # Issue 410 - elif study_name == "issue-410": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_issue_410(trial: optuna.Trial) -> float: - trial.suggest_categorical("resample_rate", ["50ms"]) - trial.suggest_categorical("channels", ["all"]) - trial.suggest_categorical("window_size", [256]) - if trial.number > 15: - raise Exception("Unexpected error") - trial.suggest_categorical("cbow", [True]) - trial.suggest_categorical("model", ["m1"]) - - trial.set_user_attr("epochs", 0) - trial.set_user_attr("deterministic", True) - if trial.number > 10: - raise Exception("unexpeccted error") - trial.set_user_attr("folder", "/path/to/folder") - trial.set_user_attr("resample_type", "foo") - trial.set_user_attr("run_id", "0001") - return 1.0 - - study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) - - # No trials single-objective study - elif study_name == "single-no-trials": - optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - # No trials multi-objective study - elif study_name == "multi-no-trials": - optuna.create_study( - study_name=study_name, - storage=storage, - directions=["minimize", "maximize"], - sampler=sampler, - ) - else: - assert False, f"No study configuration of {study_name} in conftest.py" - - return storage - - -def get_free_port() -> int: - tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - tcp.bind(("", 0)) - _, port = tcp.getsockname() - tcp.close() - return port - - -@pytest.fixture(scope="session", params=study_names) -def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage: - study_name = request.param - storage = make_dummy_storage(study_name) - return storage - - -@pytest.fixture(scope="session") -def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: - addr = "127.0.0.1" - port = get_free_port() - app = wsgi(storage) - httpd = make_server(addr, port, app) - thread = threading.Thread(target=httpd.serve_forever) - thread.start() - - def stop_server() -> None: - httpd.shutdown() - httpd.server_close() - thread.join() - - request.addfinalizer(stop_server) - - return f"http://{addr}:{port}/dashboard" - - @pytest.fixture(scope="session") def browser_context_args(browser_context_args: dict) -> dict: return { diff --git a/e2e_tests/mock_db.py b/e2e_tests/mock_db.py new file mode 100644 index 00000000..b6d14734 --- /dev/null +++ b/e2e_tests/mock_db.py @@ -0,0 +1,35 @@ +import socket +import threading +from wsgiref.simple_server import make_server + +import optuna +from optuna_dashboard import wsgi +import pytest + + +def get_free_port() -> int: + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.bind(("", 0)) + _, port = tcp.getsockname() + tcp.close() + return port + + +def make_test_server( + request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage +) -> str: + addr = "127.0.0.1" + port = get_free_port() + app = wsgi(storage) + httpd = make_server(addr, port, app) + thread = threading.Thread(target=httpd.serve_forever) + thread.start() + + def stop_server() -> None: + httpd.shutdown() + httpd.server_close() + thread.join() + + request.addfinalizer(stop_server) + + return f"http://{addr}:{port}/dashboard" diff --git a/e2e_tests/test_usecases/__init__.py b/e2e_tests/test_usecases/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/e2e_tests/test_usecases/test_study_history.py b/e2e_tests/test_usecases/test_study_history.py index f34d35e6..aef0fa1b 100644 --- a/e2e_tests/test_usecases/test_study_history.py +++ b/e2e_tests/test_usecases/test_study_history.py @@ -1,5 +1,35 @@ import optuna from playwright.sync_api import Page +import pytest + +from ..mock_db import make_test_server + + +def make_dummy_storage() -> optuna.storages.InMemoryStorage: + storage = optuna.storages.InMemoryStorage() + sampler = optuna.samplers.RandomSampler(seed=0) + + study = optuna.create_study(study_name="single", storage=storage, sampler=sampler) + + def objective_single(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective_single, n_trials=50) + + return storage + + +@pytest.fixture +def storage() -> optuna.storages.InMemoryStorage: + storage = make_dummy_storage() + return storage + + +@pytest.fixture +def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: + return make_test_server(request, storage) def test_history_xaxis_click( diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 706ec2f3..568ebab3 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,5 +1,225 @@ import optuna from playwright.sync_api import Page +import pytest + +from .mock_db import make_test_server + + +study_names = [ + "single", + "single-trial", + "single-1-param", + "single-dynamic", + "single-inf", + "multi-objective", + "multi-dynamic", + "single-pruned-without-report", + "single-inf-report", + "issue-410", + "single-no-trials", + "multi-no-trials", +] + + +def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage: + storage = optuna.storages.InMemoryStorage() + sampler = optuna.samplers.RandomSampler(seed=0) + + # Sinble objective study + if study_name == "single": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) + + def objective_single(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective_single, n_trials=50) + + # A single objective study with a single trial + # Refs: https://github.com/optuna/optuna-dashboard/issues/401 + elif study_name == "single-trial": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) + + def objective_single(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective_single, n_trials=1) + + # Single-objective study with 1 parameter + elif study_name == "single-1-param": + study = optuna.create_study( + study_name=study_name, storage=storage, direction="maximize", sampler=sampler + ) + + def objective_single_with_1param(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + return -((x1 - 2) ** 2) + + study.optimize(objective_single_with_1param, n_trials=50) + + # Single-objective study with dynamic search space + elif study_name == "single-dynamic": + study = optuna.create_study( + study_name=study_name, storage=storage, direction="maximize", sampler=sampler + ) + + def objective_single_dynamic(trial: optuna.Trial) -> float: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + return (trial.suggest_float("x1", 0, 10) - 2) ** 2 + else: + return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) + + study.optimize(objective_single_dynamic, n_trials=50) + + # Single objective study with 'inf', '-inf', or 'nan' value + elif study_name == "single-inf": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) + + def objective_single_inf(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + return float("inf") + elif trial.number % 3 == 1: + return float("-inf") + else: + return x**2 + + study.optimize(objective_single_inf, n_trials=50) + + # Multi-objective study + elif study_name == "multi-objective": + study = optuna.create_study( + study_name=study_name, + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective_multi(trial: optuna.Trial) -> tuple[float, float]: + x = trial.suggest_float("x", 0, 5) + y = trial.suggest_float("y", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + + study.optimize(objective_multi, n_trials=50) + + # Multi-objective study with dynamic search space + elif study_name == "multi-dynamic": + study = optuna.create_study( + study_name=study_name, + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + x = trial.suggest_float("x1", 0, 5) + y = trial.suggest_float("y1", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + else: + x = trial.suggest_float("x2", 0, 5) + y = trial.suggest_float("y2", 0, 3) + v0 = 2 * x**2 + 2 * y**2 + v1 = (x - 2) ** 2 + (y - 3) ** 2 + return v0, v1 + + study.optimize(objective_multi_dynamic, n_trials=50) + + # Pruning with no intermediate values + elif study_name == "single-pruned-without-report": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) + + def objective_prune_without_report(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -15, 30) + y = trial.suggest_float("y", -15, 30) + v = x**2 + y**2 + if v > 100: + raise optuna.TrialPruned() + return v + + study.optimize(objective_prune_without_report, n_trials=100) + + # Single objective pruned after reported 'inf', '-inf', or 'nan' + elif study_name == "single-inf-report": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) + + def objective_single_inf_report(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + trial.report(float("inf"), 1) + elif trial.number % 3 == 1: + trial.report(float("-inf"), 1) + else: + trial.report(float("nan"), 1) + + if x > 0: + raise optuna.TrialPruned() + else: + return x**2 + + study.optimize(objective_single_inf_report, n_trials=50) + + # Issue 410 + elif study_name == "issue-410": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) + + def objective_issue_410(trial: optuna.Trial) -> float: + trial.suggest_categorical("resample_rate", ["50ms"]) + trial.suggest_categorical("channels", ["all"]) + trial.suggest_categorical("window_size", [256]) + if trial.number > 15: + raise Exception("Unexpected error") + trial.suggest_categorical("cbow", [True]) + trial.suggest_categorical("model", ["m1"]) + + trial.set_user_attr("epochs", 0) + trial.set_user_attr("deterministic", True) + if trial.number > 10: + raise Exception("unexpeccted error") + trial.set_user_attr("folder", "/path/to/folder") + trial.set_user_attr("resample_type", "foo") + trial.set_user_attr("run_id", "0001") + return 1.0 + + study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) + + # No trials single-objective study + elif study_name == "single-no-trials": + optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) + + # No trials multi-objective study + elif study_name == "multi-no-trials": + optuna.create_study( + study_name=study_name, + storage=storage, + directions=["minimize", "maximize"], + sampler=sampler, + ) + else: + assert False, f"No study configuration of {study_name} in conftest.py" + + return storage + + +@pytest.fixture(params=study_names) +def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage: + study_name = request.param + storage = make_dummy_storage(study_name) + return storage + + +@pytest.fixture(params=study_names) +def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: + return make_test_server(request, storage) def test_study_list( From 9296fc2bbeaa7c1dad0f11d6d706b3cb27f2f9a6 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 20 Aug 2023 16:29:53 +0900 Subject: [PATCH 02/11] Rename some functions --- e2e_tests/{mock_db.py => test_server.py} | 0 e2e_tests/test_usecases/test_study_history.py | 6 +++--- e2e_tests/visual_regression_test.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) rename e2e_tests/{mock_db.py => test_server.py} (100%) diff --git a/e2e_tests/mock_db.py b/e2e_tests/test_server.py similarity index 100% rename from e2e_tests/mock_db.py rename to e2e_tests/test_server.py diff --git a/e2e_tests/test_usecases/test_study_history.py b/e2e_tests/test_usecases/test_study_history.py index aef0fa1b..e37b8f5a 100644 --- a/e2e_tests/test_usecases/test_study_history.py +++ b/e2e_tests/test_usecases/test_study_history.py @@ -2,10 +2,10 @@ import optuna from playwright.sync_api import Page import pytest -from ..mock_db import make_test_server +from ..test_server import make_test_server -def make_dummy_storage() -> optuna.storages.InMemoryStorage: +def make_test_storage() -> optuna.storages.InMemoryStorage: storage = optuna.storages.InMemoryStorage() sampler = optuna.samplers.RandomSampler(seed=0) @@ -23,7 +23,7 @@ def make_dummy_storage() -> optuna.storages.InMemoryStorage: @pytest.fixture def storage() -> optuna.storages.InMemoryStorage: - storage = make_dummy_storage() + storage = make_test_storage() return storage diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 568ebab3..0ffd448b 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -2,7 +2,7 @@ import optuna from playwright.sync_api import Page import pytest -from .mock_db import make_test_server +from .test_server import make_test_server study_names = [ @@ -21,7 +21,7 @@ study_names = [ ] -def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage: +def make_test_storage(study_name: str) -> optuna.storages.InMemoryStorage: storage = optuna.storages.InMemoryStorage() sampler = optuna.samplers.RandomSampler(seed=0) @@ -213,7 +213,7 @@ def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage: @pytest.fixture(params=study_names) def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage: study_name = request.param - storage = make_dummy_storage(study_name) + storage = make_test_storage(study_name) return storage From c05d0bd866d395647f9724c7b60ce2c508890c6e Mon Sep 17 00:00:00 2001 From: Contramundum Date: Thu, 31 Aug 2023 17:31:12 +0900 Subject: [PATCH 03/11] Add should_generate --- .../preferential-optimization/generator.py | 2 +- optuna_dashboard/preferential/_study.py | 35 ++++++++++++++++++- .../preferential/_system_attrs.py | 13 +++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index cfbbade2..d429b5db 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -35,7 +35,7 @@ def main() -> NoReturn: while True: # If n_comparison "best" trials (that are not reported bad) exists, # the generator waits for human evaluation. - if len(study.best_trials) >= n_comparison: + if study.should_generate(): time.sleep(0.1) # Avoid busy-loop continue diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index fba9b4cd..5a6d20b1 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -14,7 +14,7 @@ from optuna.trial import FrozenTrial from optuna.trial import TrialState from optuna_dashboard.preferential._system_attrs import get_preferences from optuna_dashboard.preferential._system_attrs import is_skipped_trial -from optuna_dashboard.preferential._system_attrs import report_preferences +from optuna_dashboard.preferential._system_attrs import report_preferences, get_n_generate, set_n_generate _logger = logging.get_logger(__name__) @@ -253,6 +253,36 @@ class PreferentialStudy: raise RuntimeError("Unexpected trial type") storage.set_trial_system_attr(trial_id, _SYSTEM_ATTR_COMPARISON_READY, True) + @property + def n_generate(self) -> int: + """Return the number of trials that should be generated and shown to user. + + :func:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns + :obj:`True` if the number of trials not reported bad and not skipped are less than + :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. + """ + system_attrs = self._study._storage.get_study_system_attrs(self._study._study_id) + return get_n_generate(system_attrs) + + def set_n_generate(self, n_generate: int) -> None: + """Set the number of trials that should be generated and shown to user. + + :func:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns + :obj:`True` if the number of trials not reported bad and not skipped are less than + :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. + """ + return set_n_generate(self._study._study_id, self._study._storage, n_generate) + + def should_generate(self) -> bool: + """Return whether the generator should generate a new trial now. + + Returns :obj:`True` if the number of trials not reported bad and not skipped are less than + :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. Users are recommended + to generate a new trial if this method returns :obj:`True`, and to wait for human evaluation + if this method returns :obj:`False`. + """ + return len(self.best_trials) < self.n_generate + def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]: preferences = get_preferences(study_id, storage) @@ -328,6 +358,9 @@ def create_study( study._storage.set_study_system_attr( study._study_id, _SYSTEM_ATTR_PREFERENTIAL_STUDY, True ) + study._storage.set_study_system_attr( + study._study_id, _SYSTEM_ATTR_N_GENERATE, 4 # Default n_generate is 4 + ) return PreferentialStudy(study) except optuna.exceptions.DuplicatedStudyError: diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 8c3de407..e683c776 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -9,6 +9,7 @@ from optuna.trial import TrialState _SYSTEM_ATTR_PREFIX_PREFERENCE = "preference:values" _SYSTEM_ATTR_PREFIX_SKIP_TRIAL = "preference:skip_trial:" +_SYSTEM_ATTR_N_GENERATE = "preference:n_generate" def report_preferences( @@ -60,3 +61,15 @@ def report_skip( def is_skipped_trial(trial_id: int, study_system_attrs: dict[str, Any]) -> bool: key = _SYSTEM_ATTR_PREFIX_SKIP_TRIAL + str(trial_id) return key in study_system_attrs + + +def get_n_generate(study_system_attrs: dict[str, Any]) -> int: + return study_system_attrs[_SYSTEM_ATTR_N_GENERATE] + +def set_n_generate(study_id: int, n_generate: int, storage: BaseStorage) -> None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_N_GENERATE, + value=n_generate, + ) + From 0421e550ac40ebf109a3654702239898d7ed8a36 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Thu, 31 Aug 2023 17:31:28 +0900 Subject: [PATCH 04/11] format --- optuna_dashboard/preferential/_study.py | 14 ++++++++------ optuna_dashboard/preferential/_system_attrs.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index 5a6d20b1..c793a982 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -12,9 +12,11 @@ from optuna.samplers import BaseSampler from optuna.samplers import RandomSampler from optuna.trial import FrozenTrial from optuna.trial import TrialState +from optuna_dashboard.preferential._system_attrs import get_n_generate from optuna_dashboard.preferential._system_attrs import get_preferences from optuna_dashboard.preferential._system_attrs import is_skipped_trial -from optuna_dashboard.preferential._system_attrs import report_preferences, get_n_generate, set_n_generate +from optuna_dashboard.preferential._system_attrs import report_preferences +from optuna_dashboard.preferential._system_attrs import set_n_generate _logger = logging.get_logger(__name__) @@ -256,26 +258,26 @@ class PreferentialStudy: @property def n_generate(self) -> int: """Return the number of trials that should be generated and shown to user. - + :func:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns :obj:`True` if the number of trials not reported bad and not skipped are less than :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. """ system_attrs = self._study._storage.get_study_system_attrs(self._study._study_id) return get_n_generate(system_attrs) - + def set_n_generate(self, n_generate: int) -> None: """Set the number of trials that should be generated and shown to user. - + :func:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns :obj:`True` if the number of trials not reported bad and not skipped are less than :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. """ return set_n_generate(self._study._study_id, self._study._storage, n_generate) - + def should_generate(self) -> bool: """Return whether the generator should generate a new trial now. - + Returns :obj:`True` if the number of trials not reported bad and not skipped are less than :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. Users are recommended to generate a new trial if this method returns :obj:`True`, and to wait for human evaluation diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index e683c776..5970f0d3 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -66,10 +66,10 @@ def is_skipped_trial(trial_id: int, study_system_attrs: dict[str, Any]) -> bool: def get_n_generate(study_system_attrs: dict[str, Any]) -> int: return study_system_attrs[_SYSTEM_ATTR_N_GENERATE] + def set_n_generate(study_id: int, n_generate: int, storage: BaseStorage) -> None: storage.set_study_system_attr( study_id=study_id, key=_SYSTEM_ATTR_N_GENERATE, value=n_generate, ) - From 824b684ca63f0503cb9eadc781ad19236265ccf1 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Thu, 31 Aug 2023 17:38:39 +0900 Subject: [PATCH 05/11] format --- examples/preferential-optimization/generator.py | 4 +--- optuna_dashboard/preferential/_study.py | 8 +++----- optuna_dashboard/preferential/_system_attrs.py | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index d429b5db..1a498fe7 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -20,8 +20,6 @@ artifact_path = os.path.join(os.path.dirname(__file__), "artifact") artifact_backend = FileSystemBackend(base_path=artifact_path) os.makedirs(artifact_path, exist_ok=True) -n_comparison = 5 - def main() -> NoReturn: study = create_study( @@ -35,7 +33,7 @@ def main() -> NoReturn: while True: # If n_comparison "best" trials (that are not reported bad) exists, # the generator waits for human evaluation. - if study.should_generate(): + if not study.should_generate(): time.sleep(0.1) # Avoid busy-loop continue diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index c793a982..16c81b66 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -280,8 +280,8 @@ class PreferentialStudy: Returns :obj:`True` if the number of trials not reported bad and not skipped are less than :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. Users are recommended - to generate a new trial if this method returns :obj:`True`, and to wait for human evaluation - if this method returns :obj:`False`. + to generate a new trial if this method returns :obj:`True`, and to wait for human + evaluation if this method returns :obj:`False`. """ return len(self.best_trials) < self.n_generate @@ -360,9 +360,7 @@ def create_study( study._storage.set_study_system_attr( study._study_id, _SYSTEM_ATTR_PREFERENTIAL_STUDY, True ) - study._storage.set_study_system_attr( - study._study_id, _SYSTEM_ATTR_N_GENERATE, 4 # Default n_generate is 4 - ) + set_n_generate(study._study_id, study._storage, 4) # Default n_generate return PreferentialStudy(study) except optuna.exceptions.DuplicatedStudyError: diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 5970f0d3..2964c9e0 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -67,7 +67,7 @@ def get_n_generate(study_system_attrs: dict[str, Any]) -> int: return study_system_attrs[_SYSTEM_ATTR_N_GENERATE] -def set_n_generate(study_id: int, n_generate: int, storage: BaseStorage) -> None: +def set_n_generate(study_id: int, storage: BaseStorage, n_generate: int) -> None: storage.set_study_system_attr( study_id=study_id, key=_SYSTEM_ATTR_N_GENERATE, From c7639b3f3f185b1b0ff7ceaddf06c2ab86f10bdb Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 1 Sep 2023 14:54:59 +0900 Subject: [PATCH 06/11] Change API --- optuna_dashboard/preferential/_study.py | 31 +++++++------------------ 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index 16c81b66..a7f494d4 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -255,26 +255,6 @@ class PreferentialStudy: raise RuntimeError("Unexpected trial type") storage.set_trial_system_attr(trial_id, _SYSTEM_ATTR_COMPARISON_READY, True) - @property - def n_generate(self) -> int: - """Return the number of trials that should be generated and shown to user. - - :func:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns - :obj:`True` if the number of trials not reported bad and not skipped are less than - :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. - """ - system_attrs = self._study._storage.get_study_system_attrs(self._study._study_id) - return get_n_generate(system_attrs) - - def set_n_generate(self, n_generate: int) -> None: - """Set the number of trials that should be generated and shown to user. - - :func:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns - :obj:`True` if the number of trials not reported bad and not skipped are less than - :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. - """ - return set_n_generate(self._study._study_id, self._study._storage, n_generate) - def should_generate(self) -> bool: """Return whether the generator should generate a new trial now. @@ -283,7 +263,7 @@ class PreferentialStudy: to generate a new trial if this method returns :obj:`True`, and to wait for human evaluation if this method returns :obj:`False`. """ - return len(self.best_trials) < self.n_generate + return len(self.best_trials) < get_n_generate(self._study._study_id, self._study._storage) def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]: @@ -306,6 +286,7 @@ def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list def create_study( *, + n_generate: int, storage: str | optuna.storages.BaseStorage | None = None, sampler: BaseSampler | None = None, study_name: str | None = None, @@ -325,6 +306,12 @@ def create_study( trial = study.ask() Args: + n_generate: + The number of active trials to keep. + :func:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns + :obj:`True` if the number of trials not reported bad and not skipped are less than + ``n_generate``. + storage: Database URL. If this argument is set to None, in-memory storage is used, and the :class:`~optuna_dashboard.preferential.PreferentialStudy` will not be persistent. @@ -360,7 +347,7 @@ def create_study( study._storage.set_study_system_attr( study._study_id, _SYSTEM_ATTR_PREFERENTIAL_STUDY, True ) - set_n_generate(study._study_id, study._storage, 4) # Default n_generate + set_n_generate(study._study_id, study._storage, n_generate) return PreferentialStudy(study) except optuna.exceptions.DuplicatedStudyError: From 64193747f24569f3bb1c332d990d585f4b2c53c9 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 1 Sep 2023 14:55:06 +0900 Subject: [PATCH 07/11] Change API --- examples/preferential-optimization/generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index 1a498fe7..e94f1d05 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -23,6 +23,7 @@ os.makedirs(artifact_path, exist_ok=True) def main() -> NoReturn: study = create_study( + n_generate=5, study_name="Preferential Optimization", storage=STORAGE_URL, sampler=PreferentialGPSampler(), From 7b71ae5695118b023589a06581d168014ced656b Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 1 Sep 2023 15:55:46 +0900 Subject: [PATCH 08/11] Fix test --- python_tests/preferential/test_study.py | 56 +++++++++++-------- .../preferential/test_system_attrs.py | 2 +- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/python_tests/preferential/test_study.py b/python_tests/preferential/test_study.py index 3e0b011c..3de57dae 100644 --- a/python_tests/preferential/test_study.py +++ b/python_tests/preferential/test_study.py @@ -25,7 +25,7 @@ from ..storage_supplier import StorageSupplier @parametrize_storages def test_study_set_and_get_user_attrs(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) study.set_user_attr("dataset", "MNIST") assert study.user_attrs["dataset"] == "MNIST" @@ -34,7 +34,7 @@ def test_study_set_and_get_user_attrs(storage_supplier: Callable[[], StorageSupp @parametrize_storages def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) assert len(study.preferences) == 0 for _ in range(2): @@ -51,7 +51,9 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli def test_study_pickle() -> None: - study_1 = create_study() + study_1 = create_study( + n_generate=4, + ) for _ in range(10): study_1.ask() assert len(study_1.trials) == 10 @@ -69,13 +71,17 @@ def test_study_pickle() -> None: def test_create_study(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: # Test creating a new study. - study = create_study(storage=storage, load_if_exists=False) + study = create_study(n_generate=4, storage=storage, load_if_exists=False) # Test `load_if_exists=True` with existing study. - create_study(study_name=study.study_name, storage=storage, load_if_exists=True) + create_study( + n_generate=4, study_name=study.study_name, storage=storage, load_if_exists=True + ) with pytest.raises(DuplicatedStudyError): - create_study(study_name=study.study_name, storage=storage, load_if_exists=False) + create_study( + n_generate=4, study_name=study.study_name, storage=storage, load_if_exists=False + ) @parametrize_storages @@ -92,7 +98,7 @@ def test_load_study(storage_supplier: Callable[[], StorageSupplier]) -> None: load_study(study_name=study_name, storage=storage) # Create a new study. - created_study = create_study(study_name=study_name, storage=storage) + created_study = create_study(n_generate=4, study_name=study_name, storage=storage) # Test loading an existing study. loaded_study = load_study(study_name=study_name, storage=storage) @@ -108,7 +114,7 @@ def test_load_study_study_name_none(storage_supplier: Callable[[], StorageSuppli study_name = str(uuid.uuid4()) - _ = create_study(study_name=study_name, storage=storage) + _ = create_study(n_generate=4, study_name=study_name, storage=storage) loaded_study = load_study(study_name=None, storage=storage) @@ -116,7 +122,7 @@ def test_load_study_study_name_none(storage_supplier: Callable[[], StorageSuppli study_name = str(uuid.uuid4()) - _ = create_study(study_name=study_name, storage=storage) + _ = create_study(n_generate=4, study_name=study_name, storage=storage) # Ambiguous study. with pytest.raises(ValueError): @@ -131,7 +137,7 @@ def test_delete_study(storage_supplier: Callable[[], StorageSupplier]) -> None: delete_study(study_name="invalid-study-name", storage=storage) # Test deleting an existing study. - study = create_study(storage=storage, load_if_exists=False) + study = create_study(n_generate=4, storage=storage, load_if_exists=False) delete_study(study_name=study.study_name, storage=storage) # Test failed to delete the study which is already deleted. @@ -141,7 +147,7 @@ def test_delete_study(storage_supplier: Callable[[], StorageSupplier]) -> None: def test_copy_study() -> None: with StorageSupplier("sqlite") as from_storage, StorageSupplier("sqlite") as to_storage: - from_study = create_study(storage=from_storage) + from_study = create_study(n_generate=4, storage=from_storage) from_study.set_user_attr("baz", "qux") for _ in range(3): trial = from_study.ask() @@ -165,8 +171,8 @@ def test_copy_study() -> None: def test_copy_study_to_study_name() -> None: with StorageSupplier("sqlite") as from_storage, StorageSupplier("sqlite") as to_storage: - from_study = create_study(study_name="foo", storage=from_storage) - _ = create_study(study_name="foo", storage=to_storage) + from_study = create_study(n_generate=4, study_name="foo", storage=from_storage) + _ = create_study(n_generate=4, study_name="foo", storage=to_storage) with pytest.raises(DuplicatedStudyError): copy_study( @@ -188,7 +194,7 @@ def test_copy_study_to_study_name() -> None: @parametrize_storages def test_add_trial(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) assert len(study.trials) == 0 trial = create_trial(value=0) @@ -198,7 +204,9 @@ def test_add_trial(storage_supplier: Callable[[], StorageSupplier]) -> None: def test_add_trial_invalid_values_length() -> None: - study = create_study() + study = create_study( + n_generate=4, + ) trial = create_trial(values=[0, 0]) with pytest.raises(ValueError): study.add_trial(trial) @@ -207,7 +215,7 @@ def test_add_trial_invalid_values_length() -> None: @parametrize_storages def test_add_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) assert len(study.trials) == 0 study.add_trials([]) @@ -220,7 +228,7 @@ def test_add_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: assert trial.number == i assert trial.value == i - other_study = create_study(storage=storage) + other_study = create_study(n_generate=4, storage=storage) other_study.add_trials(study.trials) assert len(other_study.trials) == 3 for i, trial in enumerate(other_study.trials): @@ -231,7 +239,7 @@ def test_add_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: @parametrize_storages def test_get_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) for _ in range(5): trial = study.ask() trial.suggest_int("x", 1, 5) @@ -256,7 +264,7 @@ def test_get_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: @parametrize_storages def test_get_trials_state_option(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) for _ in range(3): trial = study.ask() study.mark_comparison_ready(trial) @@ -286,7 +294,9 @@ def test_get_trials_state_option(storage_supplier: Callable[[], StorageSupplier] def test_ask() -> None: - study = create_study() + study = create_study( + n_generate=4, + ) trial = study.ask() assert isinstance(trial, Trial) @@ -298,7 +308,9 @@ def test_ask_fixed_search_space() -> None: "y": distributions.CategoricalDistribution(["bacon", "spam"]), } - study = create_study() + study = create_study( + n_generate=4, + ) trial = study.ask(fixed_distributions=fixed_distributions) params = trial.params @@ -312,7 +324,7 @@ def test_report_preferences_from_another_process() -> None: with StorageSupplier("sqlite") as storage: # Create a study and ask for a new trial. - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) study.ask() study.ask() diff --git a/python_tests/preferential/test_system_attrs.py b/python_tests/preferential/test_system_attrs.py index 10448d48..d36a3fef 100644 --- a/python_tests/preferential/test_system_attrs.py +++ b/python_tests/preferential/test_system_attrs.py @@ -13,7 +13,7 @@ from ..storage_supplier import StorageSupplier @parametrize_storages def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = optuna.create_study(storage=storage) + study = optuna.create_study(n_generate=4, storage=storage) study.ask() study.ask() From 0a3987f811d66b76188ad3587d25042b056fce21 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 3 Sep 2023 15:28:39 +0900 Subject: [PATCH 09/11] Follow review comments --- e2e_tests/visual_regression_test.py | 499 +++++++++++++++------------- 1 file changed, 275 insertions(+), 224 deletions(-) diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 0ffd448b..d83fbb5f 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,3 +1,5 @@ +from typing import Callable + import optuna from playwright.sync_api import Page import pytest @@ -5,231 +7,268 @@ import pytest from .test_server import make_test_server -study_names = [ - "single", - "single-trial", - "single-1-param", - "single-dynamic", - "single-inf", - "multi-objective", - "multi-dynamic", - "single-pruned-without-report", - "single-inf-report", - "issue-410", - "single-no-trials", - "multi-no-trials", -] - - -def make_test_storage(study_name: str) -> optuna.storages.InMemoryStorage: +@pytest.fixture +def storage() -> optuna.storages.InMemoryStorage: storage = optuna.storages.InMemoryStorage() - sampler = optuna.samplers.RandomSampler(seed=0) - - # Sinble objective study - if study_name == "single": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 - - study.optimize(objective_single, n_trials=50) - - # A single objective study with a single trial - # Refs: https://github.com/optuna/optuna-dashboard/issues/401 - elif study_name == "single-trial": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 - - study.optimize(objective_single, n_trials=1) - - # Single-objective study with 1 parameter - elif study_name == "single-1-param": - study = optuna.create_study( - study_name=study_name, storage=storage, direction="maximize", sampler=sampler - ) - - def objective_single_with_1param(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - return -((x1 - 2) ** 2) - - study.optimize(objective_single_with_1param, n_trials=50) - - # Single-objective study with dynamic search space - elif study_name == "single-dynamic": - study = optuna.create_study( - study_name=study_name, storage=storage, direction="maximize", sampler=sampler - ) - - def objective_single_dynamic(trial: optuna.Trial) -> float: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - return (trial.suggest_float("x1", 0, 10) - 2) ** 2 - else: - return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) - - study.optimize(objective_single_dynamic, n_trials=50) - - # Single objective study with 'inf', '-inf', or 'nan' value - elif study_name == "single-inf": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single_inf(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - return float("inf") - elif trial.number % 3 == 1: - return float("-inf") - else: - return x**2 - - study.optimize(objective_single_inf, n_trials=50) - - # Multi-objective study - elif study_name == "multi-objective": - study = optuna.create_study( - study_name=study_name, - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) - - def objective_multi(trial: optuna.Trial) -> tuple[float, float]: - x = trial.suggest_float("x", 0, 5) - y = trial.suggest_float("y", 0, 3) - v0 = 4 * x**2 + 4 * y**2 - v1 = (x - 5) ** 2 + (y - 5) ** 2 - return v0, v1 - - study.optimize(objective_multi, n_trials=50) - - # Multi-objective study with dynamic search space - elif study_name == "multi-dynamic": - study = optuna.create_study( - study_name=study_name, - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) - - def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - x = trial.suggest_float("x1", 0, 5) - y = trial.suggest_float("y1", 0, 3) - v0 = 4 * x**2 + 4 * y**2 - v1 = (x - 5) ** 2 + (y - 5) ** 2 - return v0, v1 - else: - x = trial.suggest_float("x2", 0, 5) - y = trial.suggest_float("y2", 0, 3) - v0 = 2 * x**2 + 2 * y**2 - v1 = (x - 2) ** 2 + (y - 3) ** 2 - return v0, v1 - - study.optimize(objective_multi_dynamic, n_trials=50) - - # Pruning with no intermediate values - elif study_name == "single-pruned-without-report": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_prune_without_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -15, 30) - y = trial.suggest_float("y", -15, 30) - v = x**2 + y**2 - if v > 100: - raise optuna.TrialPruned() - return v - - study.optimize(objective_prune_without_report, n_trials=100) - - # Single objective pruned after reported 'inf', '-inf', or 'nan' - elif study_name == "single-inf-report": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single_inf_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - trial.report(float("inf"), 1) - elif trial.number % 3 == 1: - trial.report(float("-inf"), 1) - else: - trial.report(float("nan"), 1) - - if x > 0: - raise optuna.TrialPruned() - else: - return x**2 - - study.optimize(objective_single_inf_report, n_trials=50) - - # Issue 410 - elif study_name == "issue-410": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_issue_410(trial: optuna.Trial) -> float: - trial.suggest_categorical("resample_rate", ["50ms"]) - trial.suggest_categorical("channels", ["all"]) - trial.suggest_categorical("window_size", [256]) - if trial.number > 15: - raise Exception("Unexpected error") - trial.suggest_categorical("cbow", [True]) - trial.suggest_categorical("model", ["m1"]) - - trial.set_user_attr("epochs", 0) - trial.set_user_attr("deterministic", True) - if trial.number > 10: - raise Exception("unexpeccted error") - trial.set_user_attr("folder", "/path/to/folder") - trial.set_user_attr("resample_type", "foo") - trial.set_user_attr("run_id", "0001") - return 1.0 - - study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) - - # No trials single-objective study - elif study_name == "single-no-trials": - optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - # No trials multi-objective study - elif study_name == "multi-no-trials": - optuna.create_study( - study_name=study_name, - storage=storage, - directions=["minimize", "maximize"], - sampler=sampler, - ) - else: - assert False, f"No study configuration of {study_name} in conftest.py" - return storage -@pytest.fixture(params=study_names) -def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage: - study_name = request.param - storage = make_test_storage(study_name) - return storage - - -@pytest.fixture(params=study_names) +@pytest.fixture def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: return make_test_server(request, storage) +def run_single_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective, n_trials=50) + return study + + +def run_single_trial_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # A single objective study with a single trial + # Refs: https://github.com/optuna/optuna-dashboard/issues/401 + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single-trial", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective, n_trials=1) + return study + + +def run_single_1param_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="single-1-param", storage=storage, direction="maximize", sampler=sampler + ) + + def objective(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + return -((x1 - 2) ** 2) + + study.optimize(objective, n_trials=50) + return study + + +def run_single_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Single-objective study with dynamic search space + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="single-dynamic", storage=storage, direction="maximize", sampler=sampler + ) + + def objective(trial: optuna.Trial) -> float: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + return (trial.suggest_float("x1", 0, 10) - 2) ** 2 + else: + return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) + + study.optimize(objective, n_trials=50) + return study + + +def run_single_inf_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Single objective study with 'inf', '-inf', or 'nan' value + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single-inf", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + return float("inf") + elif trial.number % 3 == 1: + return float("-inf") + else: + return x**2 + + study.optimize(objective, n_trials=50) + return study + + +def run_multi_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Multi-objective study + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="multi-objective", + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective(trial: optuna.Trial) -> tuple[float, float]: + x = trial.suggest_float("x", 0, 5) + y = trial.suggest_float("y", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + + study.optimize(objective, n_trials=50) + return study + + +def run_multi_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Multi-objective study with dynamic search space + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="multi-dynamic", + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective(trial: optuna.Trial) -> tuple[float, float]: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + x = trial.suggest_float("x1", 0, 5) + y = trial.suggest_float("y1", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + else: + x = trial.suggest_float("x2", 0, 5) + y = trial.suggest_float("y2", 0, 3) + v0 = 2 * x**2 + 2 * y**2 + v1 = (x - 2) ** 2 + (y - 3) ** 2 + return v0, v1 + + study.optimize(objective, n_trials=50) + return study + + +def run_single_pruned_without_report_objective_study( + storage: optuna.storages.InMemoryStorage, +) -> optuna.Study: + # Pruning with no intermediate values + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="single-pruned-without-report", storage=storage, sampler=sampler + ) + + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -15, 30) + y = trial.suggest_float("y", -15, 30) + v = x**2 + y**2 + if v > 100: + raise optuna.TrialPruned() + return v + + study.optimize(objective, n_trials=100) + return study + + +def run_single_inf_report_objective_study( + storage: optuna.storages.InMemoryStorage, +) -> optuna.Study: + # Single objective pruned after reported 'inf', '-inf', or 'nan' + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single-inf-report", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + trial.report(float("inf"), 1) + elif trial.number % 3 == 1: + trial.report(float("-inf"), 1) + else: + trial.report(float("nan"), 1) + + if x > 0: + raise optuna.TrialPruned() + else: + return x**2 + + study.optimize(objective, n_trials=50) + return study + + +def run_issue_410_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Issue 410 + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="issue-410", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + trial.suggest_categorical("resample_rate", ["50ms"]) + trial.suggest_categorical("channels", ["all"]) + trial.suggest_categorical("window_size", [256]) + if trial.number > 15: + raise Exception("Unexpected error") + trial.suggest_categorical("cbow", [True]) + trial.suggest_categorical("model", ["m1"]) + + trial.set_user_attr("epochs", 0) + trial.set_user_attr("deterministic", True) + if trial.number > 10: + raise Exception("unexpeccted error") + trial.set_user_attr("folder", "/path/to/folder") + trial.set_user_attr("resample_type", "foo") + trial.set_user_attr("run_id", "0001") + return 1.0 + + study.optimize(objective, n_trials=20, catch=(Exception,)) + return study + + +def run_single_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # No trials single-objective study + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single-no-trials", storage=storage, sampler=sampler) + + return study + + +def run_multi_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # No trials multi-objective study + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="multi-no-trials", + storage=storage, + directions=["minimize", "maximize"], + sampler=sampler, + ) + return study + + +parameterize_studies = pytest.mark.parametrize( + "run_study", + [ + run_single_objective_study, + run_single_trial_objective_study, + run_single_1param_objective_study, + run_single_dynamic_objective_study, + run_single_inf_objective_study, + run_multi_objective_study, + run_multi_dynamic_objective_study, + run_single_pruned_without_report_objective_study, + run_single_inf_report_objective_study, + run_issue_410_objective_study, + run_single_no_trials_objective_study, + run_multi_no_trials_objective_study, + ], +) + + +@parameterize_studies def test_study_list( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name page.goto(server_url) page.click(f"a[href='/dashboard/studies/{study_id}']") @@ -242,14 +281,17 @@ def test_study_list( assert study_name in title +@parameterize_studies def test_study_analytics( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name url = f"{server_url}/studies/{study_id}" page.goto(url) @@ -263,14 +305,17 @@ def test_study_analytics( assert study_name in title +@parameterize_studies def test_trial_list( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name url = f"{server_url}/studies/{study_id}" page.goto(url) @@ -284,14 +329,17 @@ def test_trial_list( assert study_name in title +@parameterize_studies def test_trial_table( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name url = f"{server_url}/studies/{study_id}" page.goto(url) @@ -305,14 +353,17 @@ def test_trial_table( assert study_name in title +@parameterize_studies def test_trial_note( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name url = f"{server_url}/studies/{study_id}" page.goto(url) From 07ea8a642a86f2e2bf002935d9bd49c2f197644e Mon Sep 17 00:00:00 2001 From: Contramundum Date: Mon, 4 Sep 2023 15:25:19 +0900 Subject: [PATCH 10/11] Fix test --- optuna_dashboard/preferential/_study.py | 2 +- python_tests/preferential/test_system_attrs.py | 2 +- python_tests/test_serializers.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index a7f494d4..ce691f42 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -263,7 +263,7 @@ class PreferentialStudy: to generate a new trial if this method returns :obj:`True`, and to wait for human evaluation if this method returns :obj:`False`. """ - return len(self.best_trials) < get_n_generate(self._study._study_id, self._study._storage) + return len(self.best_trials) < get_n_generate(self._study.system_attrs) def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]: diff --git a/python_tests/preferential/test_system_attrs.py b/python_tests/preferential/test_system_attrs.py index d36a3fef..10448d48 100644 --- a/python_tests/preferential/test_system_attrs.py +++ b/python_tests/preferential/test_system_attrs.py @@ -13,7 +13,7 @@ from ..storage_supplier import StorageSupplier @parametrize_storages def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = optuna.create_study(n_generate=4, storage=storage) + study = optuna.create_study(storage=storage) study.ask() study.ask() diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index a75db32d..a90e0de7 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -24,7 +24,7 @@ def test_serialize_dict() -> None: def test_get_study_detail_is_preferential() -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) study_summaries = get_study_summaries(storage) assert len(study_summaries) == 1 @@ -46,7 +46,7 @@ def test_get_study_detail_is_not_preferential() -> None: def test_get_study_summary_is_preferential() -> None: storage = optuna.storages.InMemoryStorage() - create_study(storage=storage) + create_study(n_generate=4, storage=storage) study_summaries = get_study_summaries(storage) assert len(study_summaries) == 1 From 7608f1bb9605ef3d52c6d5e09226d2ec110a21a8 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Mon, 4 Sep 2023 18:21:38 +0900 Subject: [PATCH 11/11] Fix test --- python_tests/test_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index ae50e29a..c0304a37 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -102,7 +102,7 @@ class APITestCase(TestCase): def test_get_best_trials_of_preferential_study(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) for _ in range(3): trial = study.ask() study.mark_comparison_ready(trial) @@ -125,7 +125,7 @@ class APITestCase(TestCase): def test_report_preference(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) for _ in range(3): trial = study.ask() study.mark_comparison_ready(trial) @@ -153,7 +153,7 @@ class APITestCase(TestCase): def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) trials: list[optuna.Trial] = [] for _ in range(3): trial = study.ask()