wassname/optuna-dashboard
1
0
Fork 0
mirror of https://github.com/wassname/optuna-dashboard.git synced 2026-09-17 12:30:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #512 from keisuke-umezawa/feature/parameterize-storages

Browse Source 
Parameterize dummy storages in e2e test
This commit is contained in:
keisuke umezawa
2023-06-25 16:47:41 +09:00
committed by GitHub
parent 3e724ba0a0 9bf1fb1932
commit 2be5d73918
2 changed files with 170 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e_tests/conftest.py
+166 -127
View File
@@ -6,184 +6,223 @@ from optuna_dashboard import wsgi
import pytest
@pytest.fixture(scope="session")
def dummy_storage() -> optuna.storages.InMemoryStorage:
study_names = [
"single",
"single-trial",
"single-1-param",
"single-dynamic",
"single-inf",
"multi-objective",
"multi-dynamic",
"single-pruned-without-report",
"single-inf-report",
"issue-410",
"single-no-trials",
"multi-no-trials",
]
def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage:
storage = optuna.storages.InMemoryStorage()
sampler = optuna.samplers.RandomSampler(seed=0)
# Single-objective study
study = optuna.create_study(study_name="single", storage=storage, sampler=sampler)
# Sinble objective study
if study_name == "single":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_single(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
x2 = trial.suggest_float("x2", 0, 10)
return (x1 - 2) ** 2 + (x2 - 5) ** 2
def objective_single(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
x2 = trial.suggest_float("x2", 0, 10)
return (x1 - 2) ** 2 + (x2 - 5) ** 2
study.optimize(objective_single, n_trials=50)
study.optimize(objective_single, n_trials=50)
# A single objective study with a single trial
# Refs: https://github.com/optuna/optuna-dashboard/issues/401
study = optuna.create_study(study_name="single-trial", storage=storage, sampler=sampler)
study.optimize(objective_single, n_trials=1)
elif study_name == "single-trial":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_single(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
x2 = trial.suggest_float("x2", 0, 10)
return (x1 - 2) ** 2 + (x2 - 5) ** 2
study.optimize(objective_single, n_trials=1)
# Single-objective study with 1 parameter
study = optuna.create_study(
study_name="single-1-param", storage=storage, direction="maximize", sampler=sampler
)
elif study_name == "single-1-param":
study = optuna.create_study(
study_name=study_name, storage=storage, direction="maximize", sampler=sampler
)
def objective_single_with_1param(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
return -((x1 - 2) ** 2)
def objective_single_with_1param(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
return -((x1 - 2) ** 2)
study.optimize(objective_single_with_1param, n_trials=50)
study.optimize(objective_single_with_1param, n_trials=50)
# Single-objective study with dynamic search space
study = optuna.create_study(
study_name="single-dynamic", storage=storage, direction="maximize", sampler=sampler
)
elif study_name == "single-dynamic":
study = optuna.create_study(
study_name=study_name, storage=storage, direction="maximize", sampler=sampler
)
def objective_single_dynamic(trial: optuna.Trial) -> float:
category = trial.suggest_categorical("category", ["foo", "bar"])
if category == "foo":
return (trial.suggest_float("x1", 0, 10) - 2) ** 2
else:
return -((trial.suggest_float("x2", -10, 0) + 5) ** 2)
def objective_single_dynamic(trial: optuna.Trial) -> float:
category = trial.suggest_categorical("category", ["foo", "bar"])
if category == "foo":
return (trial.suggest_float("x1", 0, 10) - 2) ** 2
else:
return -((trial.suggest_float("x2", -10, 0) + 5) ** 2)
study.optimize(objective_single_dynamic, n_trials=50)
study.optimize(objective_single_dynamic, n_trials=50)
# Single objective study with 'inf', '-inf', or 'nan' value
study = optuna.create_study(study_name="single-inf", storage=storage, sampler=sampler)
elif study_name == "single-inf":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_single_inf(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -10, 10)
if trial.number % 3 == 0:
return float("inf")
elif trial.number % 3 == 1:
return float("-inf")
else:
return x**2
def objective_single_inf(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -10, 10)
if trial.number % 3 == 0:
return float("inf")
elif trial.number % 3 == 1:
return float("-inf")
else:
return x**2
study.optimize(objective_single_inf, n_trials=50)
study.optimize(objective_single_inf, n_trials=50)
# Multi-objective study
study = optuna.create_study(
study_name="multi-objective",
storage=storage,
directions=["minimize", "minimize"],
sampler=sampler,
)
elif study_name == "multi-objective":
study = optuna.create_study(
study_name=study_name,
storage=storage,
directions=["minimize", "minimize"],
sampler=sampler,
)
def objective_multi(trial: optuna.Trial) -> tuple[float, float]:
x = trial.suggest_float("x", 0, 5)
y = trial.suggest_float("y", 0, 3)
v0 = 4 * x**2 + 4 * y**2
v1 = (x - 5) ** 2 + (y - 5) ** 2
return v0, v1
study.optimize(objective_multi, n_trials=50)
# Multi-objective study with dynamic search space
study = optuna.create_study(
study_name="multi-dynamic",
storage=storage,
directions=["minimize", "minimize"],
sampler=sampler,
)
def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]:
category = trial.suggest_categorical("category", ["foo", "bar"])
if category == "foo":
x = trial.suggest_float("x1", 0, 5)
y = trial.suggest_float("y1", 0, 3)
def objective_multi(trial: optuna.Trial) -> tuple[float, float]:
x = trial.suggest_float("x", 0, 5)
y = trial.suggest_float("y", 0, 3)
v0 = 4 * x**2 + 4 * y**2
v1 = (x - 5) ** 2 + (y - 5) ** 2
return v0, v1
else:
x = trial.suggest_float("x2", 0, 5)
y = trial.suggest_float("y2", 0, 3)
v0 = 2 * x**2 + 2 * y**2
v1 = (x - 2) ** 2 + (y - 3) ** 2
return v0, v1
study.optimize(objective_multi_dynamic, n_trials=50)
study.optimize(objective_multi, n_trials=50)
# Multi-objective study with dynamic search space
elif study_name == "multi-dynamic":
study = optuna.create_study(
study_name=study_name,
storage=storage,
directions=["minimize", "minimize"],
sampler=sampler,
)
def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]:
category = trial.suggest_categorical("category", ["foo", "bar"])
if category == "foo":
x = trial.suggest_float("x1", 0, 5)
y = trial.suggest_float("y1", 0, 3)
v0 = 4 * x**2 + 4 * y**2
v1 = (x - 5) ** 2 + (y - 5) ** 2
return v0, v1
else:
x = trial.suggest_float("x2", 0, 5)
y = trial.suggest_float("y2", 0, 3)
v0 = 2 * x**2 + 2 * y**2
v1 = (x - 2) ** 2 + (y - 3) ** 2
return v0, v1
study.optimize(objective_multi_dynamic, n_trials=50)
# Pruning with no intermediate values
study = optuna.create_study(
study_name="single-pruned-without-report", storage=storage, sampler=sampler
)
elif study_name == "single-pruned-without-report":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_prune_without_report(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -15, 30)
y = trial.suggest_float("y", -15, 30)
v = x**2 + y**2
if v > 100:
raise optuna.TrialPruned()
return v
def objective_prune_without_report(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -15, 30)
y = trial.suggest_float("y", -15, 30)
v = x**2 + y**2
if v > 100:
raise optuna.TrialPruned()
return v
study.optimize(objective_prune_without_report, n_trials=100)
study.optimize(objective_prune_without_report, n_trials=100)
# Single objective pruned after reported 'inf', '-inf', or 'nan'
study = optuna.create_study(study_name="single-inf-report", storage=storage, sampler=sampler)
elif study_name == "single-inf-report":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_single_inf_report(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -10, 10)
if trial.number % 3 == 0:
trial.report(float("inf"), 1)
elif trial.number % 3 == 1:
trial.report(float("-inf"), 1)
else:
trial.report(float("nan"), 1)
def objective_single_inf_report(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -10, 10)
if trial.number % 3 == 0:
trial.report(float("inf"), 1)
elif trial.number % 3 == 1:
trial.report(float("-inf"), 1)
else:
trial.report(float("nan"), 1)
if x > 0:
raise optuna.TrialPruned()
else:
return x**2
if x > 0:
raise optuna.TrialPruned()
else:
return x**2
study.optimize(objective_single_inf_report, n_trials=50)
study.optimize(objective_single_inf_report, n_trials=50)
# Issue 410
study = optuna.create_study(study_name="Issue 410", storage=storage, sampler=sampler)
elif study_name == "issue-410":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_issue_410(trial: optuna.Trial) -> float:
trial.suggest_categorical("resample_rate", ["50ms"])
trial.suggest_categorical("channels", ["all"])
trial.suggest_categorical("window_size", [256])
if trial.number > 15:
raise Exception("Unexpected error")
trial.suggest_categorical("cbow", [True])
trial.suggest_categorical("model", ["m1"])
def objective_issue_410(trial: optuna.Trial) -> float:
trial.suggest_categorical("resample_rate", ["50ms"])
trial.suggest_categorical("channels", ["all"])
trial.suggest_categorical("window_size", [256])
if trial.number > 15:
raise Exception("Unexpected error")
trial.suggest_categorical("cbow", [True])
trial.suggest_categorical("model", ["m1"])
trial.set_user_attr("epochs", 0)
trial.set_user_attr("deterministic", True)
if trial.number > 10:
raise Exception("unexpeccted error")
trial.set_user_attr("folder", "/path/to/folder")
trial.set_user_attr("resample_type", "foo")
trial.set_user_attr("run_id", "0001")
return 1.0
trial.set_user_attr("epochs", 0)
trial.set_user_attr("deterministic", True)
if trial.number > 10:
raise Exception("unexpeccted error")
trial.set_user_attr("folder", "/path/to/folder")
trial.set_user_attr("resample_type", "foo")
trial.set_user_attr("run_id", "0001")
return 1.0
study.optimize(objective_issue_410, n_trials=20, catch=(Exception,))
study.optimize(objective_issue_410, n_trials=20, catch=(Exception,))
# No trials single-objective study
optuna.create_study(study_name="single-no-trials", storage=storage, sampler=sampler)
elif study_name == "single-no-trials":
optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
# No trials multi-objective study
optuna.create_study(
study_name="multi-no-trials",
storage=storage,
directions=["minimize", "maximize"],
sampler=sampler,
)
elif study_name == "multi-no-trials":
optuna.create_study(
study_name=study_name,
storage=storage,
directions=["minimize", "maximize"],
sampler=sampler,
)
else:
assert False, f"No study configuration of {study_name} in conftest.py"
return storage
@pytest.fixture(scope="session", params=study_names)
def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage:
study_name = request.param
storage = make_dummy_storage(study_name)
return storage
@pytest.fixture(scope="session")
def server_url(
request: pytest.FixtureRequest, dummy_storage: optuna.storages.InMemoryStorage
) -> str:
def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str:
addr = "127.0.0.1"
port = 38080
app = wsgi(dummy_storage)
app = wsgi(storage)
httpd = make_server(addr, port, app)
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
e2e_tests/visual_regression_test.py
+4 -7
View File
@@ -1,21 +1,18 @@
import optuna
from playwright.sync_api import Page
import pytest
@pytest.mark.parametrize("study_id", range(10))
def test_study_list(
study_id: int,
page: Page,
dummy_storage: optuna.storages.InMemoryStorage,
storage: optuna.storages.InMemoryStorage,
server_url: str,
) -> None:
page.set_viewport_size({"width": 1000, "height": 3000})
summaries = optuna.get_all_study_summaries(dummy_storage)
study_ids = {s._study_id: s.study_name for s in summaries}
summaries = optuna.get_all_study_summaries(storage)
study_id = summaries[0]._study_id
study_name = summaries[0].study_name
study_name = study_ids[study_id]
page.goto(server_url)
page.click(f"a[href='/dashboard/studies/{study_id}']")