From 9bf1fb19326266540e9493a3f993bba1d61dbef0 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 25 Jun 2023 16:41:56 +0900 Subject: [PATCH] Parameterize dummy storages --- e2e_tests/conftest.py | 293 ++++++++++++++++------------ e2e_tests/visual_regression_test.py | 11 +- 2 files changed, 170 insertions(+), 134 deletions(-) diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 5ce9d91c..b9b8e36f 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -6,184 +6,223 @@ from optuna_dashboard import wsgi import pytest -@pytest.fixture(scope="session") -def dummy_storage() -> optuna.storages.InMemoryStorage: +study_names = [ + "single", + "single-trial", + "single-1-param", + "single-dynamic", + "single-inf", + "multi-objective", + "multi-dynamic", + "single-pruned-without-report", + "single-inf-report", + "issue-410", + "single-no-trials", + "multi-no-trials", +] + + +def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage: storage = optuna.storages.InMemoryStorage() sampler = optuna.samplers.RandomSampler(seed=0) - # Single-objective study - study = optuna.create_study(study_name="single", storage=storage, sampler=sampler) + # Sinble objective study + if study_name == "single": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_single(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 + def objective_single(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 - study.optimize(objective_single, n_trials=50) + study.optimize(objective_single, n_trials=50) # A single objective study with a single trial # Refs: https://github.com/optuna/optuna-dashboard/issues/401 - study = optuna.create_study(study_name="single-trial", storage=storage, sampler=sampler) - study.optimize(objective_single, n_trials=1) + elif study_name == "single-trial": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) + + def objective_single(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective_single, n_trials=1) # Single-objective study with 1 parameter - study = optuna.create_study( - study_name="single-1-param", storage=storage, direction="maximize", sampler=sampler - ) + elif study_name == "single-1-param": + study = optuna.create_study( + study_name=study_name, storage=storage, direction="maximize", sampler=sampler + ) - def objective_single_with_1param(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - return -((x1 - 2) ** 2) + def objective_single_with_1param(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + return -((x1 - 2) ** 2) - study.optimize(objective_single_with_1param, n_trials=50) + study.optimize(objective_single_with_1param, n_trials=50) # Single-objective study with dynamic search space - study = optuna.create_study( - study_name="single-dynamic", storage=storage, direction="maximize", sampler=sampler - ) + elif study_name == "single-dynamic": + study = optuna.create_study( + study_name=study_name, storage=storage, direction="maximize", sampler=sampler + ) - def objective_single_dynamic(trial: optuna.Trial) -> float: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - return (trial.suggest_float("x1", 0, 10) - 2) ** 2 - else: - return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) + def objective_single_dynamic(trial: optuna.Trial) -> float: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + return (trial.suggest_float("x1", 0, 10) - 2) ** 2 + else: + return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) - study.optimize(objective_single_dynamic, n_trials=50) + study.optimize(objective_single_dynamic, n_trials=50) # Single objective study with 'inf', '-inf', or 'nan' value - study = optuna.create_study(study_name="single-inf", storage=storage, sampler=sampler) + elif study_name == "single-inf": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_single_inf(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - return float("inf") - elif trial.number % 3 == 1: - return float("-inf") - else: - return x**2 + def objective_single_inf(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + return float("inf") + elif trial.number % 3 == 1: + return float("-inf") + else: + return x**2 - study.optimize(objective_single_inf, n_trials=50) + study.optimize(objective_single_inf, n_trials=50) # Multi-objective study - study = optuna.create_study( - study_name="multi-objective", - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) + elif study_name == "multi-objective": + study = optuna.create_study( + study_name=study_name, + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) - def objective_multi(trial: optuna.Trial) -> tuple[float, float]: - x = trial.suggest_float("x", 0, 5) - y = trial.suggest_float("y", 0, 3) - v0 = 4 * x**2 + 4 * y**2 - v1 = (x - 5) ** 2 + (y - 5) ** 2 - return v0, v1 - - study.optimize(objective_multi, n_trials=50) - - # Multi-objective study with dynamic search space - study = optuna.create_study( - study_name="multi-dynamic", - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) - - def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - x = trial.suggest_float("x1", 0, 5) - y = trial.suggest_float("y1", 0, 3) + def objective_multi(trial: optuna.Trial) -> tuple[float, float]: + x = trial.suggest_float("x", 0, 5) + y = trial.suggest_float("y", 0, 3) v0 = 4 * x**2 + 4 * y**2 v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 - else: - x = trial.suggest_float("x2", 0, 5) - y = trial.suggest_float("y2", 0, 3) - v0 = 2 * x**2 + 2 * y**2 - v1 = (x - 2) ** 2 + (y - 3) ** 2 - return v0, v1 - study.optimize(objective_multi_dynamic, n_trials=50) + study.optimize(objective_multi, n_trials=50) + + # Multi-objective study with dynamic search space + elif study_name == "multi-dynamic": + study = optuna.create_study( + study_name=study_name, + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + x = trial.suggest_float("x1", 0, 5) + y = trial.suggest_float("y1", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + else: + x = trial.suggest_float("x2", 0, 5) + y = trial.suggest_float("y2", 0, 3) + v0 = 2 * x**2 + 2 * y**2 + v1 = (x - 2) ** 2 + (y - 3) ** 2 + return v0, v1 + + study.optimize(objective_multi_dynamic, n_trials=50) # Pruning with no intermediate values - study = optuna.create_study( - study_name="single-pruned-without-report", storage=storage, sampler=sampler - ) + elif study_name == "single-pruned-without-report": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_prune_without_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -15, 30) - y = trial.suggest_float("y", -15, 30) - v = x**2 + y**2 - if v > 100: - raise optuna.TrialPruned() - return v + def objective_prune_without_report(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -15, 30) + y = trial.suggest_float("y", -15, 30) + v = x**2 + y**2 + if v > 100: + raise optuna.TrialPruned() + return v - study.optimize(objective_prune_without_report, n_trials=100) + study.optimize(objective_prune_without_report, n_trials=100) # Single objective pruned after reported 'inf', '-inf', or 'nan' - study = optuna.create_study(study_name="single-inf-report", storage=storage, sampler=sampler) + elif study_name == "single-inf-report": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_single_inf_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - trial.report(float("inf"), 1) - elif trial.number % 3 == 1: - trial.report(float("-inf"), 1) - else: - trial.report(float("nan"), 1) + def objective_single_inf_report(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + trial.report(float("inf"), 1) + elif trial.number % 3 == 1: + trial.report(float("-inf"), 1) + else: + trial.report(float("nan"), 1) - if x > 0: - raise optuna.TrialPruned() - else: - return x**2 + if x > 0: + raise optuna.TrialPruned() + else: + return x**2 - study.optimize(objective_single_inf_report, n_trials=50) + study.optimize(objective_single_inf_report, n_trials=50) # Issue 410 - study = optuna.create_study(study_name="Issue 410", storage=storage, sampler=sampler) + elif study_name == "issue-410": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_issue_410(trial: optuna.Trial) -> float: - trial.suggest_categorical("resample_rate", ["50ms"]) - trial.suggest_categorical("channels", ["all"]) - trial.suggest_categorical("window_size", [256]) - if trial.number > 15: - raise Exception("Unexpected error") - trial.suggest_categorical("cbow", [True]) - trial.suggest_categorical("model", ["m1"]) + def objective_issue_410(trial: optuna.Trial) -> float: + trial.suggest_categorical("resample_rate", ["50ms"]) + trial.suggest_categorical("channels", ["all"]) + trial.suggest_categorical("window_size", [256]) + if trial.number > 15: + raise Exception("Unexpected error") + trial.suggest_categorical("cbow", [True]) + trial.suggest_categorical("model", ["m1"]) - trial.set_user_attr("epochs", 0) - trial.set_user_attr("deterministic", True) - if trial.number > 10: - raise Exception("unexpeccted error") - trial.set_user_attr("folder", "/path/to/folder") - trial.set_user_attr("resample_type", "foo") - trial.set_user_attr("run_id", "0001") - return 1.0 + trial.set_user_attr("epochs", 0) + trial.set_user_attr("deterministic", True) + if trial.number > 10: + raise Exception("unexpeccted error") + trial.set_user_attr("folder", "/path/to/folder") + trial.set_user_attr("resample_type", "foo") + trial.set_user_attr("run_id", "0001") + return 1.0 - study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) + study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) # No trials single-objective study - optuna.create_study(study_name="single-no-trials", storage=storage, sampler=sampler) + elif study_name == "single-no-trials": + optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) # No trials multi-objective study - optuna.create_study( - study_name="multi-no-trials", - storage=storage, - directions=["minimize", "maximize"], - sampler=sampler, - ) + elif study_name == "multi-no-trials": + optuna.create_study( + study_name=study_name, + storage=storage, + directions=["minimize", "maximize"], + sampler=sampler, + ) + else: + assert False, f"No study configuration of {study_name} in conftest.py" + + return storage + + +@pytest.fixture(scope="session", params=study_names) +def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage: + study_name = request.param + storage = make_dummy_storage(study_name) return storage @pytest.fixture(scope="session") -def server_url( - request: pytest.FixtureRequest, dummy_storage: optuna.storages.InMemoryStorage -) -> str: +def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: addr = "127.0.0.1" port = 38080 - app = wsgi(dummy_storage) + app = wsgi(storage) httpd = make_server(addr, port, app) thread = threading.Thread(target=httpd.serve_forever) thread.start() diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index b4091bc4..42b2190c 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,21 +1,18 @@ import optuna from playwright.sync_api import Page -import pytest -@pytest.mark.parametrize("study_id", range(10)) def test_study_list( - study_id: int, page: Page, - dummy_storage: optuna.storages.InMemoryStorage, + storage: optuna.storages.InMemoryStorage, server_url: str, ) -> None: page.set_viewport_size({"width": 1000, "height": 3000}) - summaries = optuna.get_all_study_summaries(dummy_storage) - study_ids = {s._study_id: s.study_name for s in summaries} + summaries = optuna.get_all_study_summaries(storage) + study_id = summaries[0]._study_id + study_name = summaries[0].study_name - study_name = study_ids[study_id] page.goto(server_url) page.click(f"a[href='/dashboard/studies/{study_id}']")