wassname/optuna-dashboard
1
0
Fork 0
mirror of https://github.com/wassname/optuna-dashboard.git synced 2026-09-09 11:28:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into preferential-history

Browse Source
This commit is contained in:
moririn2528
2023-09-06 11:27:22 +09:00
parent 2ed5887652 1bdc18e88d
commit b1ad5eeff3
13 changed files with 426 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1 -1
View File
@@ -45,7 +45,7 @@ docs: docs/conf.py $(RST_FILES)
.PHONY: fmt
fmt:
npm run fmt
black ./optuna_dashboard/ ./python_tests/
black ./optuna_dashboard/ ./python_tests/ ./e2e_tests/
isort .
.PHONY: clean
e2e_tests/__init__.py
View File
e2e_tests/conftest.py
-245
View File
@@ -1,251 +1,6 @@
import socket
import threading
from wsgiref.simple_server import make_server
import optuna
from optuna_dashboard import wsgi
import pytest
study_names = [
"single",
"single-trial",
"single-1-param",
"single-dynamic",
"single-inf",
"multi-objective",
"multi-dynamic",
"single-pruned-without-report",
"single-inf-report",
"issue-410",
"single-no-trials",
"multi-no-trials",
]
def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage:
storage = optuna.storages.InMemoryStorage()
sampler = optuna.samplers.RandomSampler(seed=0)
# Sinble objective study
if study_name == "single":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_single(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
x2 = trial.suggest_float("x2", 0, 10)
return (x1 - 2) ** 2 + (x2 - 5) ** 2
study.optimize(objective_single, n_trials=50)
# A single objective study with a single trial
# Refs: https://github.com/optuna/optuna-dashboard/issues/401
elif study_name == "single-trial":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_single(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
x2 = trial.suggest_float("x2", 0, 10)
return (x1 - 2) ** 2 + (x2 - 5) ** 2
study.optimize(objective_single, n_trials=1)
# Single-objective study with 1 parameter
elif study_name == "single-1-param":
study = optuna.create_study(
study_name=study_name, storage=storage, direction="maximize", sampler=sampler
)
def objective_single_with_1param(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
return -((x1 - 2) ** 2)
study.optimize(objective_single_with_1param, n_trials=50)
# Single-objective study with dynamic search space
elif study_name == "single-dynamic":
study = optuna.create_study(
study_name=study_name, storage=storage, direction="maximize", sampler=sampler
)
def objective_single_dynamic(trial: optuna.Trial) -> float:
category = trial.suggest_categorical("category", ["foo", "bar"])
if category == "foo":
return (trial.suggest_float("x1", 0, 10) - 2) ** 2
else:
return -((trial.suggest_float("x2", -10, 0) + 5) ** 2)
study.optimize(objective_single_dynamic, n_trials=50)
# Single objective study with 'inf', '-inf', or 'nan' value
elif study_name == "single-inf":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_single_inf(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -10, 10)
if trial.number % 3 == 0:
return float("inf")
elif trial.number % 3 == 1:
return float("-inf")
else:
return x**2
study.optimize(objective_single_inf, n_trials=50)
# Multi-objective study
elif study_name == "multi-objective":
study = optuna.create_study(
study_name=study_name,
storage=storage,
directions=["minimize", "minimize"],
sampler=sampler,
)
def objective_multi(trial: optuna.Trial) -> tuple[float, float]:
x = trial.suggest_float("x", 0, 5)
y = trial.suggest_float("y", 0, 3)
v0 = 4 * x**2 + 4 * y**2
v1 = (x - 5) ** 2 + (y - 5) ** 2
return v0, v1
study.optimize(objective_multi, n_trials=50)
# Multi-objective study with dynamic search space
elif study_name == "multi-dynamic":
study = optuna.create_study(
study_name=study_name,
storage=storage,
directions=["minimize", "minimize"],
sampler=sampler,
)
def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]:
category = trial.suggest_categorical("category", ["foo", "bar"])
if category == "foo":
x = trial.suggest_float("x1", 0, 5)
y = trial.suggest_float("y1", 0, 3)
v0 = 4 * x**2 + 4 * y**2
v1 = (x - 5) ** 2 + (y - 5) ** 2
return v0, v1
else:
x = trial.suggest_float("x2", 0, 5)
y = trial.suggest_float("y2", 0, 3)
v0 = 2 * x**2 + 2 * y**2
v1 = (x - 2) ** 2 + (y - 3) ** 2
return v0, v1
study.optimize(objective_multi_dynamic, n_trials=50)
# Pruning with no intermediate values
elif study_name == "single-pruned-without-report":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_prune_without_report(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -15, 30)
y = trial.suggest_float("y", -15, 30)
v = x**2 + y**2
if v > 100:
raise optuna.TrialPruned()
return v
study.optimize(objective_prune_without_report, n_trials=100)
# Single objective pruned after reported 'inf', '-inf', or 'nan'
elif study_name == "single-inf-report":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_single_inf_report(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -10, 10)
if trial.number % 3 == 0:
trial.report(float("inf"), 1)
elif trial.number % 3 == 1:
trial.report(float("-inf"), 1)
else:
trial.report(float("nan"), 1)
if x > 0:
raise optuna.TrialPruned()
else:
return x**2
study.optimize(objective_single_inf_report, n_trials=50)
# Issue 410
elif study_name == "issue-410":
study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
def objective_issue_410(trial: optuna.Trial) -> float:
trial.suggest_categorical("resample_rate", ["50ms"])
trial.suggest_categorical("channels", ["all"])
trial.suggest_categorical("window_size", [256])
if trial.number > 15:
raise Exception("Unexpected error")
trial.suggest_categorical("cbow", [True])
trial.suggest_categorical("model", ["m1"])
trial.set_user_attr("epochs", 0)
trial.set_user_attr("deterministic", True)
if trial.number > 10:
raise Exception("unexpeccted error")
trial.set_user_attr("folder", "/path/to/folder")
trial.set_user_attr("resample_type", "foo")
trial.set_user_attr("run_id", "0001")
return 1.0
study.optimize(objective_issue_410, n_trials=20, catch=(Exception,))
# No trials single-objective study
elif study_name == "single-no-trials":
optuna.create_study(study_name=study_name, storage=storage, sampler=sampler)
# No trials multi-objective study
elif study_name == "multi-no-trials":
optuna.create_study(
study_name=study_name,
storage=storage,
directions=["minimize", "maximize"],
sampler=sampler,
)
else:
assert False, f"No study configuration of {study_name} in conftest.py"
return storage
def get_free_port() -> int:
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.bind(("", 0))
_, port = tcp.getsockname()
tcp.close()
return port
@pytest.fixture(scope="session", params=study_names)
def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage:
study_name = request.param
storage = make_dummy_storage(study_name)
return storage
@pytest.fixture(scope="session")
def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str:
addr = "127.0.0.1"
port = get_free_port()
app = wsgi(storage)
httpd = make_server(addr, port, app)
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
def stop_server() -> None:
httpd.shutdown()
httpd.server_close()
thread.join()
request.addfinalizer(stop_server)
return f"http://{addr}:{port}/dashboard"
@pytest.fixture(scope="session")
def browser_context_args(browser_context_args: dict) -> dict:
return {
e2e_tests/test_server.py
+35
View File
@@ -0,0 +1,35 @@
import socket
import threading
from wsgiref.simple_server import make_server
import optuna
from optuna_dashboard import wsgi
import pytest
def get_free_port() -> int:
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.bind(("", 0))
_, port = tcp.getsockname()
tcp.close()
return port
def make_test_server(
request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage
) -> str:
addr = "127.0.0.1"
port = get_free_port()
app = wsgi(storage)
httpd = make_server(addr, port, app)
thread = threading.Thread(target=httpd.serve_forever)
thread.start()
def stop_server() -> None:
httpd.shutdown()
httpd.server_close()
thread.join()
request.addfinalizer(stop_server)
return f"http://{addr}:{port}/dashboard"
e2e_tests/test_usecases/__init__.py
View File
e2e_tests/test_usecases/test_study_history.py
+30
View File
@@ -1,5 +1,35 @@
import optuna
from playwright.sync_api import Page
import pytest
from ..test_server import make_test_server
def make_test_storage() -> optuna.storages.InMemoryStorage:
storage = optuna.storages.InMemoryStorage()
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(study_name="single", storage=storage, sampler=sampler)
def objective_single(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
x2 = trial.suggest_float("x2", 0, 10)
return (x1 - 2) ** 2 + (x2 - 5) ** 2
study.optimize(objective_single, n_trials=50)
return storage
@pytest.fixture
def storage() -> optuna.storages.InMemoryStorage:
storage = make_test_storage()
return storage
@pytest.fixture
def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str:
return make_test_server(request, storage)
def test_history_xaxis_click(
e2e_tests/visual_regression_test.py
+286 -15
View File
@@ -1,15 +1,274 @@
from typing import Callable
import optuna
from playwright.sync_api import Page
import pytest
from .test_server import make_test_server
@pytest.fixture
def storage() -> optuna.storages.InMemoryStorage:
storage = optuna.storages.InMemoryStorage()
return storage
@pytest.fixture
def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str:
return make_test_server(request, storage)
def run_single_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(study_name="single", storage=storage, sampler=sampler)
def objective(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
x2 = trial.suggest_float("x2", 0, 10)
return (x1 - 2) ** 2 + (x2 - 5) ** 2
study.optimize(objective, n_trials=50)
return study
def run_single_trial_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
# A single objective study with a single trial
# Refs: https://github.com/optuna/optuna-dashboard/issues/401
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(study_name="single-trial", storage=storage, sampler=sampler)
def objective(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
x2 = trial.suggest_float("x2", 0, 10)
return (x1 - 2) ** 2 + (x2 - 5) ** 2
study.optimize(objective, n_trials=1)
return study
def run_single_1param_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(
study_name="single-1-param", storage=storage, direction="maximize", sampler=sampler
)
def objective(trial: optuna.Trial) -> float:
x1 = trial.suggest_float("x1", 0, 10)
return -((x1 - 2) ** 2)
study.optimize(objective, n_trials=50)
return study
def run_single_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
# Single-objective study with dynamic search space
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(
study_name="single-dynamic", storage=storage, direction="maximize", sampler=sampler
)
def objective(trial: optuna.Trial) -> float:
category = trial.suggest_categorical("category", ["foo", "bar"])
if category == "foo":
return (trial.suggest_float("x1", 0, 10) - 2) ** 2
else:
return -((trial.suggest_float("x2", -10, 0) + 5) ** 2)
study.optimize(objective, n_trials=50)
return study
def run_single_inf_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
# Single objective study with 'inf', '-inf', or 'nan' value
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(study_name="single-inf", storage=storage, sampler=sampler)
def objective(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -10, 10)
if trial.number % 3 == 0:
return float("inf")
elif trial.number % 3 == 1:
return float("-inf")
else:
return x**2
study.optimize(objective, n_trials=50)
return study
def run_multi_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
# Multi-objective study
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(
study_name="multi-objective",
storage=storage,
directions=["minimize", "minimize"],
sampler=sampler,
)
def objective(trial: optuna.Trial) -> tuple[float, float]:
x = trial.suggest_float("x", 0, 5)
y = trial.suggest_float("y", 0, 3)
v0 = 4 * x**2 + 4 * y**2
v1 = (x - 5) ** 2 + (y - 5) ** 2
return v0, v1
study.optimize(objective, n_trials=50)
return study
def run_multi_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
# Multi-objective study with dynamic search space
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(
study_name="multi-dynamic",
storage=storage,
directions=["minimize", "minimize"],
sampler=sampler,
)
def objective(trial: optuna.Trial) -> tuple[float, float]:
category = trial.suggest_categorical("category", ["foo", "bar"])
if category == "foo":
x = trial.suggest_float("x1", 0, 5)
y = trial.suggest_float("y1", 0, 3)
v0 = 4 * x**2 + 4 * y**2
v1 = (x - 5) ** 2 + (y - 5) ** 2
return v0, v1
else:
x = trial.suggest_float("x2", 0, 5)
y = trial.suggest_float("y2", 0, 3)
v0 = 2 * x**2 + 2 * y**2
v1 = (x - 2) ** 2 + (y - 3) ** 2
return v0, v1
study.optimize(objective, n_trials=50)
return study
def run_single_pruned_without_report_objective_study(
storage: optuna.storages.InMemoryStorage,
) -> optuna.Study:
# Pruning with no intermediate values
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(
study_name="single-pruned-without-report", storage=storage, sampler=sampler
)
def objective(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -15, 30)
y = trial.suggest_float("y", -15, 30)
v = x**2 + y**2
if v > 100:
raise optuna.TrialPruned()
return v
study.optimize(objective, n_trials=100)
return study
def run_single_inf_report_objective_study(
storage: optuna.storages.InMemoryStorage,
) -> optuna.Study:
# Single objective pruned after reported 'inf', '-inf', or 'nan'
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(study_name="single-inf-report", storage=storage, sampler=sampler)
def objective(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -10, 10)
if trial.number % 3 == 0:
trial.report(float("inf"), 1)
elif trial.number % 3 == 1:
trial.report(float("-inf"), 1)
else:
trial.report(float("nan"), 1)
if x > 0:
raise optuna.TrialPruned()
else:
return x**2
study.optimize(objective, n_trials=50)
return study
def run_issue_410_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
# Issue 410
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(study_name="issue-410", storage=storage, sampler=sampler)
def objective(trial: optuna.Trial) -> float:
trial.suggest_categorical("resample_rate", ["50ms"])
trial.suggest_categorical("channels", ["all"])
trial.suggest_categorical("window_size", [256])
if trial.number > 15:
raise Exception("Unexpected error")
trial.suggest_categorical("cbow", [True])
trial.suggest_categorical("model", ["m1"])
trial.set_user_attr("epochs", 0)
trial.set_user_attr("deterministic", True)
if trial.number > 10:
raise Exception("unexpeccted error")
trial.set_user_attr("folder", "/path/to/folder")
trial.set_user_attr("resample_type", "foo")
trial.set_user_attr("run_id", "0001")
return 1.0
study.optimize(objective, n_trials=20, catch=(Exception,))
return study
def run_single_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
# No trials single-objective study
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(study_name="single-no-trials", storage=storage, sampler=sampler)
return study
def run_multi_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study:
# No trials multi-objective study
sampler = optuna.samplers.RandomSampler(seed=0)
study = optuna.create_study(
study_name="multi-no-trials",
storage=storage,
directions=["minimize", "maximize"],
sampler=sampler,
)
return study
parameterize_studies = pytest.mark.parametrize(
"run_study",
[
run_single_objective_study,
run_single_trial_objective_study,
run_single_1param_objective_study,
run_single_dynamic_objective_study,
run_single_inf_objective_study,
run_multi_objective_study,
run_multi_dynamic_objective_study,
run_single_pruned_without_report_objective_study,
run_single_inf_report_objective_study,
run_issue_410_objective_study,
run_single_no_trials_objective_study,
run_multi_no_trials_objective_study,
],
)
@parameterize_studies
def test_study_list(
page: Page,
storage: optuna.storages.InMemoryStorage,
server_url: str,
run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study],
) -> None:
summaries = optuna.get_all_study_summaries(storage)
study_id = summaries[0]._study_id
study_name = summaries[0].study_name
study = run_study(storage)
study_id = study._study_id
study_name = study.study_name
page.goto(server_url)
page.click(f"a[href='/dashboard/studies/{study_id}']")
@@ -22,14 +281,17 @@ def test_study_list(
assert study_name in title
@parameterize_studies
def test_study_analytics(
page: Page,
storage: optuna.storages.InMemoryStorage,
server_url: str,
run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study],
) -> None:
summaries = optuna.get_all_study_summaries(storage)
study_id = summaries[0]._study_id
study_name = summaries[0].study_name
study = run_study(storage)
study_id = study._study_id
study_name = study.study_name
url = f"{server_url}/studies/{study_id}"
page.goto(url)
@@ -43,14 +305,17 @@ def test_study_analytics(
assert study_name in title
@parameterize_studies
def test_trial_list(
page: Page,
storage: optuna.storages.InMemoryStorage,
server_url: str,
run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study],
) -> None:
summaries = optuna.get_all_study_summaries(storage)
study_id = summaries[0]._study_id
study_name = summaries[0].study_name
study = run_study(storage)
study_id = study._study_id
study_name = study.study_name
url = f"{server_url}/studies/{study_id}"
page.goto(url)
@@ -64,14 +329,17 @@ def test_trial_list(
assert study_name in title
@parameterize_studies
def test_trial_table(
page: Page,
storage: optuna.storages.InMemoryStorage,
server_url: str,
run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study],
) -> None:
summaries = optuna.get_all_study_summaries(storage)
study_id = summaries[0]._study_id
study_name = summaries[0].study_name
study = run_study(storage)
study_id = study._study_id
study_name = study.study_name
url = f"{server_url}/studies/{study_id}"
page.goto(url)
@@ -85,14 +353,17 @@ def test_trial_table(
assert study_name in title
@parameterize_studies
def test_trial_note(
page: Page,
storage: optuna.storages.InMemoryStorage,
server_url: str,
run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study],
) -> None:
summaries = optuna.get_all_study_summaries(storage)
study_id = summaries[0]._study_id
study_name = summaries[0].study_name
study = run_study(storage)
study_id = study._study_id
study_name = study.study_name
url = f"{server_url}/studies/{study_id}"
page.goto(url)
examples/preferential-optimization/generator.py
+2 -3
View File
@@ -20,11 +20,10 @@ artifact_path = os.path.join(os.path.dirname(__file__), "artifact")
artifact_backend = FileSystemBackend(base_path=artifact_path)
os.makedirs(artifact_path, exist_ok=True)
n_comparison = 5
def main() -> NoReturn:
study = create_study(
n_generate=5,
study_name="Preferential Optimization",
storage=STORAGE_URL,
sampler=PreferentialGPSampler(),
@@ -35,7 +34,7 @@ def main() -> NoReturn:
while True:
# If n_comparison "best" trials (that are not reported bad) exists,
# the generator waits for human evaluation.
if len(study.best_trials) >= n_comparison:
if not study.should_generate():
time.sleep(0.1) # Avoid busy-loop
continue
optuna_dashboard/preferential/_study.py
+20
View File
@@ -12,9 +12,11 @@ from optuna.samplers import BaseSampler
from optuna.samplers import RandomSampler
from optuna.trial import FrozenTrial
from optuna.trial import TrialState
from optuna_dashboard.preferential._system_attrs import get_n_generate
from optuna_dashboard.preferential._system_attrs import get_preferences
from optuna_dashboard.preferential._system_attrs import is_skipped_trial
from optuna_dashboard.preferential._system_attrs import report_preferences
from optuna_dashboard.preferential._system_attrs import set_n_generate
_logger = logging.get_logger(__name__)
@@ -253,6 +255,16 @@ class PreferentialStudy:
raise RuntimeError("Unexpected trial type")
storage.set_trial_system_attr(trial_id, _SYSTEM_ATTR_COMPARISON_READY, True)
def should_generate(self) -> bool:
"""Return whether the generator should generate a new trial now.
Returns :obj:`True` if the number of trials not reported bad and not skipped are less than
:attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. Users are recommended
to generate a new trial if this method returns :obj:`True`, and to wait for human
evaluation if this method returns :obj:`False`.
"""
return len(self.best_trials) < get_n_generate(self._study.system_attrs)
def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]:
preferences = get_preferences(study_id, storage)
@@ -274,6 +286,7 @@ def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list
def create_study(
*,
n_generate: int,
storage: str | optuna.storages.BaseStorage | None = None,
sampler: BaseSampler | None = None,
study_name: str | None = None,
@@ -293,6 +306,12 @@ def create_study(
trial = study.ask()
Args:
n_generate:
The number of active trials to keep.
:func:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns
:obj:`True` if the number of trials not reported bad and not skipped are less than
``n_generate``.
storage:
Database URL. If this argument is set to None, in-memory storage is used, and the
:class:`~optuna_dashboard.preferential.PreferentialStudy` will not be persistent.
@@ -328,6 +347,7 @@ def create_study(
study._storage.set_study_system_attr(
study._study_id, _SYSTEM_ATTR_PREFERENTIAL_STUDY, True
)
set_n_generate(study._study_id, study._storage, n_generate)
return PreferentialStudy(study)
except optuna.exceptions.DuplicatedStudyError:
optuna_dashboard/preferential/_system_attrs.py
+13
View File
@@ -9,6 +9,7 @@ from optuna.trial import TrialState
_SYSTEM_ATTR_PREFIX_PREFERENCE = "preference:values"
_SYSTEM_ATTR_PREFIX_SKIP_TRIAL = "preference:skip_trial:"
_SYSTEM_ATTR_N_GENERATE = "preference:n_generate"
def report_preferences(
@@ -62,3 +63,15 @@ def report_skip(
def is_skipped_trial(trial_id: int, study_system_attrs: dict[str, Any]) -> bool:
key = _SYSTEM_ATTR_PREFIX_SKIP_TRIAL + str(trial_id)
return key in study_system_attrs
def get_n_generate(study_system_attrs: dict[str, Any]) -> int:
return study_system_attrs[_SYSTEM_ATTR_N_GENERATE]
def set_n_generate(study_id: int, storage: BaseStorage, n_generate: int) -> None:
storage.set_study_system_attr(
study_id=study_id,
key=_SYSTEM_ATTR_N_GENERATE,
value=n_generate,
)
python_tests/preferential/test_study.py
+34 -22
View File
@@ -25,7 +25,7 @@ from ..storage_supplier import StorageSupplier
@parametrize_storages
def test_study_set_and_get_user_attrs(storage_supplier: Callable[[], StorageSupplier]) -> None:
with storage_supplier() as storage:
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
study.set_user_attr("dataset", "MNIST")
assert study.user_attrs["dataset"] == "MNIST"
@@ -34,7 +34,7 @@ def test_study_set_and_get_user_attrs(storage_supplier: Callable[[], StorageSupp
@parametrize_storages
def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSupplier]) -> None:
with storage_supplier() as storage:
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
assert len(study.preferences) == 0
for _ in range(2):
@@ -51,7 +51,9 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli
def test_study_pickle() -> None:
study_1 = create_study()
study_1 = create_study(
n_generate=4,
)
for _ in range(10):
study_1.ask()
assert len(study_1.trials) == 10
@@ -69,13 +71,17 @@ def test_study_pickle() -> None:
def test_create_study(storage_supplier: Callable[[], StorageSupplier]) -> None:
with storage_supplier() as storage:
# Test creating a new study.
study = create_study(storage=storage, load_if_exists=False)
study = create_study(n_generate=4, storage=storage, load_if_exists=False)
# Test `load_if_exists=True` with existing study.
create_study(study_name=study.study_name, storage=storage, load_if_exists=True)
create_study(
n_generate=4, study_name=study.study_name, storage=storage, load_if_exists=True
)
with pytest.raises(DuplicatedStudyError):
create_study(study_name=study.study_name, storage=storage, load_if_exists=False)
create_study(
n_generate=4, study_name=study.study_name, storage=storage, load_if_exists=False
)
@parametrize_storages
@@ -92,7 +98,7 @@ def test_load_study(storage_supplier: Callable[[], StorageSupplier]) -> None:
load_study(study_name=study_name, storage=storage)
# Create a new study.
created_study = create_study(study_name=study_name, storage=storage)
created_study = create_study(n_generate=4, study_name=study_name, storage=storage)
# Test loading an existing study.
loaded_study = load_study(study_name=study_name, storage=storage)
@@ -108,7 +114,7 @@ def test_load_study_study_name_none(storage_supplier: Callable[[], StorageSuppli
study_name = str(uuid.uuid4())
_ = create_study(study_name=study_name, storage=storage)
_ = create_study(n_generate=4, study_name=study_name, storage=storage)
loaded_study = load_study(study_name=None, storage=storage)
@@ -116,7 +122,7 @@ def test_load_study_study_name_none(storage_supplier: Callable[[], StorageSuppli
study_name = str(uuid.uuid4())
_ = create_study(study_name=study_name, storage=storage)
_ = create_study(n_generate=4, study_name=study_name, storage=storage)
# Ambiguous study.
with pytest.raises(ValueError):
@@ -131,7 +137,7 @@ def test_delete_study(storage_supplier: Callable[[], StorageSupplier]) -> None:
delete_study(study_name="invalid-study-name", storage=storage)
# Test deleting an existing study.
study = create_study(storage=storage, load_if_exists=False)
study = create_study(n_generate=4, storage=storage, load_if_exists=False)
delete_study(study_name=study.study_name, storage=storage)
# Test failed to delete the study which is already deleted.
@@ -141,7 +147,7 @@ def test_delete_study(storage_supplier: Callable[[], StorageSupplier]) -> None:
def test_copy_study() -> None:
with StorageSupplier("sqlite") as from_storage, StorageSupplier("sqlite") as to_storage:
from_study = create_study(storage=from_storage)
from_study = create_study(n_generate=4, storage=from_storage)
from_study.set_user_attr("baz", "qux")
for _ in range(3):
trial = from_study.ask()
@@ -165,8 +171,8 @@ def test_copy_study() -> None:
def test_copy_study_to_study_name() -> None:
with StorageSupplier("sqlite") as from_storage, StorageSupplier("sqlite") as to_storage:
from_study = create_study(study_name="foo", storage=from_storage)
_ = create_study(study_name="foo", storage=to_storage)
from_study = create_study(n_generate=4, study_name="foo", storage=from_storage)
_ = create_study(n_generate=4, study_name="foo", storage=to_storage)
with pytest.raises(DuplicatedStudyError):
copy_study(
@@ -188,7 +194,7 @@ def test_copy_study_to_study_name() -> None:
@parametrize_storages
def test_add_trial(storage_supplier: Callable[[], StorageSupplier]) -> None:
with storage_supplier() as storage:
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
assert len(study.trials) == 0
trial = create_trial(value=0)
@@ -198,7 +204,9 @@ def test_add_trial(storage_supplier: Callable[[], StorageSupplier]) -> None:
def test_add_trial_invalid_values_length() -> None:
study = create_study()
study = create_study(
n_generate=4,
)
trial = create_trial(values=[0, 0])
with pytest.raises(ValueError):
study.add_trial(trial)
@@ -207,7 +215,7 @@ def test_add_trial_invalid_values_length() -> None:
@parametrize_storages
def test_add_trials(storage_supplier: Callable[[], StorageSupplier]) -> None:
with storage_supplier() as storage:
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
assert len(study.trials) == 0
study.add_trials([])
@@ -220,7 +228,7 @@ def test_add_trials(storage_supplier: Callable[[], StorageSupplier]) -> None:
assert trial.number == i
assert trial.value == i
other_study = create_study(storage=storage)
other_study = create_study(n_generate=4, storage=storage)
other_study.add_trials(study.trials)
assert len(other_study.trials) == 3
for i, trial in enumerate(other_study.trials):
@@ -231,7 +239,7 @@ def test_add_trials(storage_supplier: Callable[[], StorageSupplier]) -> None:
@parametrize_storages
def test_get_trials(storage_supplier: Callable[[], StorageSupplier]) -> None:
with storage_supplier() as storage:
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
for _ in range(5):
trial = study.ask()
trial.suggest_int("x", 1, 5)
@@ -256,7 +264,7 @@ def test_get_trials(storage_supplier: Callable[[], StorageSupplier]) -> None:
@parametrize_storages
def test_get_trials_state_option(storage_supplier: Callable[[], StorageSupplier]) -> None:
with storage_supplier() as storage:
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
for _ in range(3):
trial = study.ask()
study.mark_comparison_ready(trial)
@@ -286,7 +294,9 @@ def test_get_trials_state_option(storage_supplier: Callable[[], StorageSupplier]
def test_ask() -> None:
study = create_study()
study = create_study(
n_generate=4,
)
trial = study.ask()
assert isinstance(trial, Trial)
@@ -298,7 +308,9 @@ def test_ask_fixed_search_space() -> None:
"y": distributions.CategoricalDistribution(["bacon", "spam"]),
}
study = create_study()
study = create_study(
n_generate=4,
)
trial = study.ask(fixed_distributions=fixed_distributions)
params = trial.params
@@ -312,7 +324,7 @@ def test_report_preferences_from_another_process() -> None:
with StorageSupplier("sqlite") as storage:
# Create a study and ask for a new trial.
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
study.ask()
study.ask()
python_tests/test_api.py
+3 -3
View File
@@ -102,7 +102,7 @@ class APITestCase(TestCase):
def test_get_best_trials_of_preferential_study(self) -> None:
storage = optuna.storages.InMemoryStorage()
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
for _ in range(3):
trial = study.ask()
study.mark_comparison_ready(trial)
@@ -125,7 +125,7 @@ class APITestCase(TestCase):
def test_report_preference(self) -> None:
storage = optuna.storages.InMemoryStorage()
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
for _ in range(3):
trial = study.ask()
study.mark_comparison_ready(trial)
@@ -183,7 +183,7 @@ class APITestCase(TestCase):
def test_skip_trial(self) -> None:
storage = optuna.storages.InMemoryStorage()
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
trials: list[optuna.Trial] = []
for _ in range(3):
trial = study.ask()
python_tests/test_serializers.py
+2 -2
View File
@@ -24,7 +24,7 @@ def test_serialize_dict() -> None:
def test_get_study_detail_is_preferential() -> None:
storage = optuna.storages.InMemoryStorage()
study = create_study(storage=storage)
study = create_study(n_generate=4, storage=storage)
study_summaries = get_study_summaries(storage)
assert len(study_summaries) == 1
@@ -46,7 +46,7 @@ def test_get_study_detail_is_not_preferential() -> None:
def test_get_study_summary_is_preferential() -> None:
storage = optuna.storages.InMemoryStorage()
create_study(storage=storage)
create_study(n_generate=4, storage=storage)
study_summaries = get_study_summaries(storage)
assert len(study_summaries) == 1