diff --git a/.github/workflows/python-coverage.yml b/.github/workflows/python-coverage.yml index 4bda6350..c96f506b 100644 --- a/.github/workflows/python-coverage.yml +++ b/.github/workflows/python-coverage.yml @@ -45,4 +45,4 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml - fail_ci_if_error: true + fail_ci_if_error: false diff --git a/Makefile b/Makefile index f925bb78..4094fd57 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ docs: docs/conf.py $(RST_FILES) .PHONY: fmt fmt: npm run fmt - black ./optuna_dashboard/ ./python_tests/ + black ./optuna_dashboard/ ./python_tests/ ./e2e_tests/ isort . .PHONY: clean diff --git a/docs/api.rst b/docs/api.rst index 09a9c14b..aadd2718 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -14,6 +14,7 @@ General APIs optuna_dashboard.wsgi optuna_dashboard.set_objective_names optuna_dashboard.save_note + optuna_dashboard.save_plotly_graph_object Human-in-the-loop ----------------- diff --git a/e2e_tests/__init__.py b/e2e_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 8d8eac3b..ad1190cb 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -1,251 +1,6 @@ -import socket -import threading -from wsgiref.simple_server import make_server - -import optuna -from optuna_dashboard import wsgi import pytest -study_names = [ - "single", - "single-trial", - "single-1-param", - "single-dynamic", - "single-inf", - "multi-objective", - "multi-dynamic", - "single-pruned-without-report", - "single-inf-report", - "issue-410", - "single-no-trials", - "multi-no-trials", -] - - -def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage: - storage = optuna.storages.InMemoryStorage() - sampler = optuna.samplers.RandomSampler(seed=0) - - # Sinble objective study - if study_name == "single": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 - - study.optimize(objective_single, n_trials=50) - - # A single objective study with a single trial - # Refs: https://github.com/optuna/optuna-dashboard/issues/401 - elif study_name == "single-trial": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 - - study.optimize(objective_single, n_trials=1) - - # Single-objective study with 1 parameter - elif study_name == "single-1-param": - study = optuna.create_study( - study_name=study_name, storage=storage, direction="maximize", sampler=sampler - ) - - def objective_single_with_1param(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - return -((x1 - 2) ** 2) - - study.optimize(objective_single_with_1param, n_trials=50) - - # Single-objective study with dynamic search space - elif study_name == "single-dynamic": - study = optuna.create_study( - study_name=study_name, storage=storage, direction="maximize", sampler=sampler - ) - - def objective_single_dynamic(trial: optuna.Trial) -> float: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - return (trial.suggest_float("x1", 0, 10) - 2) ** 2 - else: - return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) - - study.optimize(objective_single_dynamic, n_trials=50) - - # Single objective study with 'inf', '-inf', or 'nan' value - elif study_name == "single-inf": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single_inf(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - return float("inf") - elif trial.number % 3 == 1: - return float("-inf") - else: - return x**2 - - study.optimize(objective_single_inf, n_trials=50) - - # Multi-objective study - elif study_name == "multi-objective": - study = optuna.create_study( - study_name=study_name, - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) - - def objective_multi(trial: optuna.Trial) -> tuple[float, float]: - x = trial.suggest_float("x", 0, 5) - y = trial.suggest_float("y", 0, 3) - v0 = 4 * x**2 + 4 * y**2 - v1 = (x - 5) ** 2 + (y - 5) ** 2 - return v0, v1 - - study.optimize(objective_multi, n_trials=50) - - # Multi-objective study with dynamic search space - elif study_name == "multi-dynamic": - study = optuna.create_study( - study_name=study_name, - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) - - def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - x = trial.suggest_float("x1", 0, 5) - y = trial.suggest_float("y1", 0, 3) - v0 = 4 * x**2 + 4 * y**2 - v1 = (x - 5) ** 2 + (y - 5) ** 2 - return v0, v1 - else: - x = trial.suggest_float("x2", 0, 5) - y = trial.suggest_float("y2", 0, 3) - v0 = 2 * x**2 + 2 * y**2 - v1 = (x - 2) ** 2 + (y - 3) ** 2 - return v0, v1 - - study.optimize(objective_multi_dynamic, n_trials=50) - - # Pruning with no intermediate values - elif study_name == "single-pruned-without-report": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_prune_without_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -15, 30) - y = trial.suggest_float("y", -15, 30) - v = x**2 + y**2 - if v > 100: - raise optuna.TrialPruned() - return v - - study.optimize(objective_prune_without_report, n_trials=100) - - # Single objective pruned after reported 'inf', '-inf', or 'nan' - elif study_name == "single-inf-report": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_single_inf_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - trial.report(float("inf"), 1) - elif trial.number % 3 == 1: - trial.report(float("-inf"), 1) - else: - trial.report(float("nan"), 1) - - if x > 0: - raise optuna.TrialPruned() - else: - return x**2 - - study.optimize(objective_single_inf_report, n_trials=50) - - # Issue 410 - elif study_name == "issue-410": - study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - def objective_issue_410(trial: optuna.Trial) -> float: - trial.suggest_categorical("resample_rate", ["50ms"]) - trial.suggest_categorical("channels", ["all"]) - trial.suggest_categorical("window_size", [256]) - if trial.number > 15: - raise Exception("Unexpected error") - trial.suggest_categorical("cbow", [True]) - trial.suggest_categorical("model", ["m1"]) - - trial.set_user_attr("epochs", 0) - trial.set_user_attr("deterministic", True) - if trial.number > 10: - raise Exception("unexpeccted error") - trial.set_user_attr("folder", "/path/to/folder") - trial.set_user_attr("resample_type", "foo") - trial.set_user_attr("run_id", "0001") - return 1.0 - - study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) - - # No trials single-objective study - elif study_name == "single-no-trials": - optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - - # No trials multi-objective study - elif study_name == "multi-no-trials": - optuna.create_study( - study_name=study_name, - storage=storage, - directions=["minimize", "maximize"], - sampler=sampler, - ) - else: - assert False, f"No study configuration of {study_name} in conftest.py" - - return storage - - -def get_free_port() -> int: - tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - tcp.bind(("", 0)) - _, port = tcp.getsockname() - tcp.close() - return port - - -@pytest.fixture(scope="session", params=study_names) -def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage: - study_name = request.param - storage = make_dummy_storage(study_name) - return storage - - -@pytest.fixture(scope="session") -def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: - addr = "127.0.0.1" - port = get_free_port() - app = wsgi(storage) - httpd = make_server(addr, port, app) - thread = threading.Thread(target=httpd.serve_forever) - thread.start() - - def stop_server() -> None: - httpd.shutdown() - httpd.server_close() - thread.join() - - request.addfinalizer(stop_server) - - return f"http://{addr}:{port}/dashboard" - - @pytest.fixture(scope="session") def browser_context_args(browser_context_args: dict) -> dict: return { diff --git a/e2e_tests/test_server.py b/e2e_tests/test_server.py new file mode 100644 index 00000000..b6d14734 --- /dev/null +++ b/e2e_tests/test_server.py @@ -0,0 +1,35 @@ +import socket +import threading +from wsgiref.simple_server import make_server + +import optuna +from optuna_dashboard import wsgi +import pytest + + +def get_free_port() -> int: + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.bind(("", 0)) + _, port = tcp.getsockname() + tcp.close() + return port + + +def make_test_server( + request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage +) -> str: + addr = "127.0.0.1" + port = get_free_port() + app = wsgi(storage) + httpd = make_server(addr, port, app) + thread = threading.Thread(target=httpd.serve_forever) + thread.start() + + def stop_server() -> None: + httpd.shutdown() + httpd.server_close() + thread.join() + + request.addfinalizer(stop_server) + + return f"http://{addr}:{port}/dashboard" diff --git a/e2e_tests/test_usecases/__init__.py b/e2e_tests/test_usecases/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/e2e_tests/test_usecases/test_study_history.py b/e2e_tests/test_usecases/test_study_history.py index f34d35e6..e37b8f5a 100644 --- a/e2e_tests/test_usecases/test_study_history.py +++ b/e2e_tests/test_usecases/test_study_history.py @@ -1,5 +1,35 @@ import optuna from playwright.sync_api import Page +import pytest + +from ..test_server import make_test_server + + +def make_test_storage() -> optuna.storages.InMemoryStorage: + storage = optuna.storages.InMemoryStorage() + sampler = optuna.samplers.RandomSampler(seed=0) + + study = optuna.create_study(study_name="single", storage=storage, sampler=sampler) + + def objective_single(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective_single, n_trials=50) + + return storage + + +@pytest.fixture +def storage() -> optuna.storages.InMemoryStorage: + storage = make_test_storage() + return storage + + +@pytest.fixture +def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: + return make_test_server(request, storage) def test_history_xaxis_click( diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 706ec2f3..d83fbb5f 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,15 +1,274 @@ +from typing import Callable + import optuna from playwright.sync_api import Page +import pytest + +from .test_server import make_test_server +@pytest.fixture +def storage() -> optuna.storages.InMemoryStorage: + storage = optuna.storages.InMemoryStorage() + return storage + + +@pytest.fixture +def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: + return make_test_server(request, storage) + + +def run_single_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective, n_trials=50) + return study + + +def run_single_trial_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # A single objective study with a single trial + # Refs: https://github.com/optuna/optuna-dashboard/issues/401 + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single-trial", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective, n_trials=1) + return study + + +def run_single_1param_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="single-1-param", storage=storage, direction="maximize", sampler=sampler + ) + + def objective(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + return -((x1 - 2) ** 2) + + study.optimize(objective, n_trials=50) + return study + + +def run_single_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Single-objective study with dynamic search space + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="single-dynamic", storage=storage, direction="maximize", sampler=sampler + ) + + def objective(trial: optuna.Trial) -> float: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + return (trial.suggest_float("x1", 0, 10) - 2) ** 2 + else: + return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) + + study.optimize(objective, n_trials=50) + return study + + +def run_single_inf_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Single objective study with 'inf', '-inf', or 'nan' value + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single-inf", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + return float("inf") + elif trial.number % 3 == 1: + return float("-inf") + else: + return x**2 + + study.optimize(objective, n_trials=50) + return study + + +def run_multi_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Multi-objective study + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="multi-objective", + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective(trial: optuna.Trial) -> tuple[float, float]: + x = trial.suggest_float("x", 0, 5) + y = trial.suggest_float("y", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + + study.optimize(objective, n_trials=50) + return study + + +def run_multi_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Multi-objective study with dynamic search space + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="multi-dynamic", + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective(trial: optuna.Trial) -> tuple[float, float]: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + x = trial.suggest_float("x1", 0, 5) + y = trial.suggest_float("y1", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + else: + x = trial.suggest_float("x2", 0, 5) + y = trial.suggest_float("y2", 0, 3) + v0 = 2 * x**2 + 2 * y**2 + v1 = (x - 2) ** 2 + (y - 3) ** 2 + return v0, v1 + + study.optimize(objective, n_trials=50) + return study + + +def run_single_pruned_without_report_objective_study( + storage: optuna.storages.InMemoryStorage, +) -> optuna.Study: + # Pruning with no intermediate values + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="single-pruned-without-report", storage=storage, sampler=sampler + ) + + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -15, 30) + y = trial.suggest_float("y", -15, 30) + v = x**2 + y**2 + if v > 100: + raise optuna.TrialPruned() + return v + + study.optimize(objective, n_trials=100) + return study + + +def run_single_inf_report_objective_study( + storage: optuna.storages.InMemoryStorage, +) -> optuna.Study: + # Single objective pruned after reported 'inf', '-inf', or 'nan' + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single-inf-report", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + trial.report(float("inf"), 1) + elif trial.number % 3 == 1: + trial.report(float("-inf"), 1) + else: + trial.report(float("nan"), 1) + + if x > 0: + raise optuna.TrialPruned() + else: + return x**2 + + study.optimize(objective, n_trials=50) + return study + + +def run_issue_410_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # Issue 410 + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="issue-410", storage=storage, sampler=sampler) + + def objective(trial: optuna.Trial) -> float: + trial.suggest_categorical("resample_rate", ["50ms"]) + trial.suggest_categorical("channels", ["all"]) + trial.suggest_categorical("window_size", [256]) + if trial.number > 15: + raise Exception("Unexpected error") + trial.suggest_categorical("cbow", [True]) + trial.suggest_categorical("model", ["m1"]) + + trial.set_user_attr("epochs", 0) + trial.set_user_attr("deterministic", True) + if trial.number > 10: + raise Exception("unexpeccted error") + trial.set_user_attr("folder", "/path/to/folder") + trial.set_user_attr("resample_type", "foo") + trial.set_user_attr("run_id", "0001") + return 1.0 + + study.optimize(objective, n_trials=20, catch=(Exception,)) + return study + + +def run_single_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # No trials single-objective study + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study(study_name="single-no-trials", storage=storage, sampler=sampler) + + return study + + +def run_multi_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: + # No trials multi-objective study + sampler = optuna.samplers.RandomSampler(seed=0) + study = optuna.create_study( + study_name="multi-no-trials", + storage=storage, + directions=["minimize", "maximize"], + sampler=sampler, + ) + return study + + +parameterize_studies = pytest.mark.parametrize( + "run_study", + [ + run_single_objective_study, + run_single_trial_objective_study, + run_single_1param_objective_study, + run_single_dynamic_objective_study, + run_single_inf_objective_study, + run_multi_objective_study, + run_multi_dynamic_objective_study, + run_single_pruned_without_report_objective_study, + run_single_inf_report_objective_study, + run_issue_410_objective_study, + run_single_no_trials_objective_study, + run_multi_no_trials_objective_study, + ], +) + + +@parameterize_studies def test_study_list( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name page.goto(server_url) page.click(f"a[href='/dashboard/studies/{study_id}']") @@ -22,14 +281,17 @@ def test_study_list( assert study_name in title +@parameterize_studies def test_study_analytics( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name url = f"{server_url}/studies/{study_id}" page.goto(url) @@ -43,14 +305,17 @@ def test_study_analytics( assert study_name in title +@parameterize_studies def test_trial_list( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name url = f"{server_url}/studies/{study_id}" page.goto(url) @@ -64,14 +329,17 @@ def test_trial_list( assert study_name in title +@parameterize_studies def test_trial_table( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name url = f"{server_url}/studies/{study_id}" page.goto(url) @@ -85,14 +353,17 @@ def test_trial_table( assert study_name in title +@parameterize_studies def test_trial_note( page: Page, storage: optuna.storages.InMemoryStorage, server_url: str, + run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: - summaries = optuna.get_all_study_summaries(storage) - study_id = summaries[0]._study_id - study_name = summaries[0].study_name + study = run_study(storage) + + study_id = study._study_id + study_name = study.study_name url = f"{server_url}/studies/{study_id}" page.goto(url) diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index cfbbade2..d8f7d3bd 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -20,11 +20,10 @@ artifact_path = os.path.join(os.path.dirname(__file__), "artifact") artifact_backend = FileSystemBackend(base_path=artifact_path) os.makedirs(artifact_path, exist_ok=True) -n_comparison = 5 - def main() -> NoReturn: study = create_study( + n_generate=5, study_name="Preferential Optimization", storage=STORAGE_URL, sampler=PreferentialGPSampler(), @@ -35,7 +34,7 @@ def main() -> NoReturn: while True: # If n_comparison "best" trials (that are not reported bad) exists, # the generator waits for human evaluation. - if len(study.best_trials) >= n_comparison: + if not study.should_generate(): time.sleep(0.1) # Avoid busy-loop continue @@ -65,9 +64,6 @@ def main() -> NoReturn: ) save_note(trial, note) - # 5. Mark comparison ready - study.mark_comparison_ready(trial) - if __name__ == "__main__": main() diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 3d363cf4..5bb3f301 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -1,5 +1,6 @@ from ._app import run_server # noqa from ._app import wsgi # noqa +from ._custom_plot_data import save_plotly_graph_object # noqa from ._form_widget import ChoiceWidget # noqa from ._form_widget import dict_to_form_widget # noqa from ._form_widget import ObjectiveChoiceWidget # noqa @@ -15,4 +16,4 @@ from ._note import get_note # noqa from ._note import save_note # noqa -__version__ = "0.12.0" +__version__ = "0.13.0b1" diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 0b69dfc7..65bb9f34 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -25,8 +25,10 @@ from . import _note as note from ._bottle_util import BottleViewReturn from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property +from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preferential_history import NewHistory from ._preferential_history import report_history from ._preferential_history import switching_history from ._rdb_migration import register_rdb_migration_route @@ -214,6 +216,8 @@ def create_app( union_user_attrs, has_intermediate_values, ) = get_cached_extra_study_property(study_id, trials) + + plotly_graph_objects = get_plotly_graph_objects(system_attrs) return serialize_study_detail( summary, best_trials, @@ -222,6 +226,7 @@ def create_app( union, union_user_attrs, has_intermediate_values, + plotly_graph_objects, ) @app.get("/api/studies//param_importances") @@ -275,7 +280,12 @@ def create_app( clicked = int(request.json.get("clicked", -1)) except ValueError: response.status = 400 - return {"reason": "Invalid request."} + return { + "reason": ( + "`candidates` should be an array of integers and " + "`clicked` should be an integer." + ) + } if clicked == -1: response.status = 400 @@ -287,11 +297,11 @@ def create_app( report_history( study_id, storage, - { - "mode": mode, - "candidates": candidates, - "clicked": clicked, - }, + NewHistory( + mode=mode, + candidates=candidates, + clicked=clicked, + ), ) response.status = 204 diff --git a/optuna_dashboard/_custom_plot_data.py b/optuna_dashboard/_custom_plot_data.py new file mode 100644 index 00000000..a4dfa4af --- /dev/null +++ b/optuna_dashboard/_custom_plot_data.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import math +from typing import TYPE_CHECKING +import uuid + +from optuna import Study + + +if TYPE_CHECKING: + from typing import Any + + from optuna.storages import BaseStorage + import plotly.graph_objs as go + + +SYSTEM_ATTR_PLOT_DATA = "dashboard:plot_data:" +SYSTEM_ATTR_MAX_LENGTH = 2045 + + +def save_plotly_graph_object( + study: Study, figure: go.Figure, *, graph_object_id: str | None = None +) -> str: + """Save the user-defined plotly's graph object to the study. + + Example: + + .. code-block:: python + + import optuna + from optuna_dashboard import save_plotly_graph_object + + def objective(trial): + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + study.optimize(objective, n_trials=100) + + figure = optuna.visualization.plot_optimization_history(study) + save_plotly_graph_object(study, figure) + + Args: + study: + Target study object. + plot_data: + The plotly's graph object to save. + graph_object_id: + Unique identifier of the graph object. If specified, the graph object is overwritten. + This must be a valid HTML id attribute value. + + Returns: + The graph object ID. + """ + if graph_object_id is not None and not is_valid_graph_object_id(graph_object_id): + raise ValueError("graph_object_id must be a valid HTML id attribute value.") + + storage = study._storage + study_id = study._study_id + + graph_object_id = graph_object_id or str(uuid.uuid4()) + key = SYSTEM_ATTR_PLOT_DATA + graph_object_id + ":" + plot_data_json_str = figure.to_json() + save_graph_object_json(storage, study_id, key, plot_data_json_str) + return graph_object_id + + +def save_graph_object_json( + storage: BaseStorage, study_id: int, key_prefix: str, plot_data_json_str: str +) -> None: + plot_data_system_attrs = split_plot_data(plot_data_json_str, key_prefix) + for k, v in plot_data_system_attrs.items(): + storage.set_study_system_attr(study_id, k, v) + + # Clear previous graph object attributes + study_system_attrs = storage.get_study_system_attrs(study_id) + all_plot_data_system_attrs = [k for k in study_system_attrs if k.startswith(key_prefix)] + if len(all_plot_data_system_attrs) > len(plot_data_system_attrs): + for i in range(len(plot_data_system_attrs), len(all_plot_data_system_attrs)): + storage.set_study_system_attr(study_id, f"{key_prefix}{i}", "") + + +def list_graph_object_ids(system_attrs: dict[str, Any]) -> list[str]: + titles = set() + for key in system_attrs: + if not key.startswith(SYSTEM_ATTR_PLOT_DATA): + continue + + s = key.split(":", maxsplit=2) # e.g. ["dashboard", "plot_data", "Optimization History:1"] + if len(s) != 3: + continue + # Please note that title may contain ":". + title = s[2].rsplit(":", maxsplit=1)[0] + titles.add(title) + return list(titles) + + +def get_plotly_graph_objects(system_attrs: dict[str, Any]) -> dict[str, str]: + graph_objects = {} + for title in list_graph_object_ids(system_attrs): + key_prefix = SYSTEM_ATTR_PLOT_DATA + title + ":" + plot_data_attrs = {k: v for k, v in system_attrs.items() if k.startswith(key_prefix)} + graph_objects[title] = concat_plot_data(plot_data_attrs, key_prefix) + return graph_objects + + +def split_plot_data(plot_data_str: str, key_prefix: str) -> dict[str, str]: + plot_data_len = len(plot_data_str) + attrs = {} + for i in range(math.ceil(plot_data_len / SYSTEM_ATTR_MAX_LENGTH)): + start = i * SYSTEM_ATTR_MAX_LENGTH + end = min((i + 1) * SYSTEM_ATTR_MAX_LENGTH, plot_data_len) + attrs[f"{key_prefix}{i}"] = plot_data_str[start:end] + return attrs + + +def concat_plot_data(plot_data_attrs: dict[str, str], key_prefix: str) -> str: + return "".join(plot_data_attrs[f"{key_prefix}{i}"] for i in range(len(plot_data_attrs))) + + +def is_valid_graph_object_id(graph_object_id: str) -> bool: + if len(graph_object_id) == 0: + return False + + # Can only contain letters [A-Za-z], numbers [0-9], hyphens ("-"), underscores ("_"), + # colons, and periods. + if not all( + "a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9" or c in ("-", "_", ":", ".") + for c in graph_object_id[1:] + ): + return False + # Unlike HTML id attribute, graph object id can begin with a letter [A-Za-z] + return True diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index f95d57d3..3b9cd3e0 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -1,158 +1,98 @@ from __future__ import annotations from dataclasses import dataclass -from dataclasses import field from datetime import datetime -from typing import Any -from typing import Literal +import json from typing import TYPE_CHECKING import uuid from optuna.storages import BaseStorage -from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE -from .preferential._system_attrs import get_preference -from .preferential._system_attrs import report_preferences +from .preferential._system_attrs import report_preferences, _SYSTEM_ATTR_PREFIX_PREFERENCE _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" if TYPE_CHECKING: + from typing import Literal from typing import TypedDict - NewHistoryJSON = TypedDict( - "NewHistoryJSON", + FeedbackMode = Literal["ChooseWorst"] + ChooseWorstHistory = TypedDict( + "ChooseWorstHistory", { - "mode": Literal["ChooseWorst"], + "mode": FeedbackMode, + "id": str, + "preference_id": str, + "timestamp": str, "candidates": list[int], "clicked": int, }, ) + History = ChooseWorstHistory @dataclass -class ChooseWorstHistory: - mode: Literal["ChooseWorst"] - uuid: str - preference_uuid: str # making it possible to remove the preference - timestamp: datetime - candidates: list[int] # a list of trial number - clicked: int # The worst trial number in the candidates. - evacuated_preference: list[tuple[int, int]] = field(default_factory=list) - # When undo the preference, this is used. Otherwise, this must be empty. - - def to_dict(self) -> dict[str, Any]: - return { - "mode": self.mode, - "uuid": self.uuid, - "preference_uuid": self.preference_uuid, - "timestamp": self.timestamp.isoformat(), - "candidates": self.candidates, - "clicked": self.clicked, - "enabled": len(self.evacuated_preference) == 0, - "evacuated_preference": self.evacuated_preference, - } - - -History = ChooseWorstHistory +class NewHistory: + mode: FeedbackMode + candidates: list[int] + clicked: int def report_history( study_id: int, storage: BaseStorage, - input_data: NewHistoryJSON, -) -> str: + input_data: NewHistory, +) -> None: preferences = [] - if input_data["mode"] == "ChooseWorst": + # TODO(moririn): Use TypeGuard after adding other history types. + if input_data.mode == "ChooseWorst": preferences = [ - (best, input_data["clicked"]) - for best in input_data["candidates"] - if best != input_data["clicked"] + (best, input_data.clicked) + for best in input_data.candidates + if best != input_data.clicked ] else: - assert False, f"Unknown mode: {input_data['mode']}" + assert False, f"Unknown data: {input_data}" - preference_uuid = report_preferences( + preference_id = report_preferences( study_id=study_id, storage=storage, preferences=preferences, ) - history_uuid = str(uuid.uuid4()) + history_id = str(uuid.uuid4()) - if input_data["mode"] == "ChooseWorst": - history = ChooseWorstHistory( - mode="ChooseWorst", - uuid=history_uuid, - preference_uuid=preference_uuid, - timestamp=datetime.now(), - candidates=input_data["candidates"], - clicked=input_data["clicked"], - ) + if input_data.mode == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": history_id, + "preference_id": preference_id, + "timestamp": datetime.now().isoformat(), + "candidates": input_data.candidates, + "clicked": input_data.clicked, + } - key = _SYSTEM_ATTR_PREFIX_HISTORY + history_uuid + key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id storage.set_study_system_attr( study_id=study_id, key=key, - value=history.to_dict(), + value=json.dumps(history), ) - return history_uuid - - -def _load_preference_history(value: Any) -> History: - choice: dict[str, Any] = value - if choice["mode"] == "ChooseWorst": - return ChooseWorstHistory( - mode="ChooseWorst", - uuid=choice["uuid"], - preference_uuid=choice["preference_uuid"], - timestamp=datetime.fromisoformat(choice["timestamp"]), - candidates=choice["candidates"], - clicked=choice["clicked"], - evacuated_preference=choice["evacuated_preference"], - ) - else: - assert False, f"Unknown mode: {choice['mode']}" - - -def load_preference_history( - uuid: str, - system_attrs: dict[str, Any], -) -> History: - value = system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, []) - return _load_preference_history(value) - - -def serialize_preference_histories( - system_attrs: dict[str, Any], -) -> list[dict[str, Any]]: - histories: list[History] = [] - for k, v in system_attrs.items(): - if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): - continue - histories.append(_load_preference_history(v)) - - histories.sort(key=lambda c: c.timestamp) - return [history.to_dict() for history in histories] def switching_history(study_id: int, storage: BaseStorage, uuid: str, enable: bool) -> None: system_attrs = storage.get_study_system_attrs(study_id) - history = load_preference_history(uuid, system_attrs) - preference = get_preference(study_id, storage, history.preference_uuid) - print(history, preference, enable) - if enable and (len(preference) > 0 or len(history.evacuated_preference) == 0): - return - if (not enable) and (len(preference) == 0 or len(history.evacuated_preference) > 0): - return - history.evacuated_preference, preference = preference, history.evacuated_preference - print(history.to_dict(), preference) - storage.set_study_system_attr( - study_id=study_id, - key=_SYSTEM_ATTR_PREFIX_HISTORY + history.uuid, - value=history.to_dict(), - ) - storage.set_study_system_attr( - study_id=study_id, - key=_SYSTEM_ATTR_PREFIX_PREFERENCE + history.preference_uuid, - value=preference, - ) + history: History = system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, None) + if enable: + preferences = [ + (best, history["clickedx"]) + for best in history["candidates"] + if best != history["clicked"] + ] + storage.set_study_system_attr( + study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], preferences + ) + else: # disable + storage.set_study_system_attr( + study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], [] + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index fc040424..cc9401b7 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import json from typing import Any from typing import TYPE_CHECKING @@ -14,15 +15,19 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names -from ._preferential_history import serialize_preference_histories +from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY +from .preferential._system_attrs import is_preference_valid if TYPE_CHECKING: from typing import Literal from typing import TypedDict + from ._preferential_history import ChooseWorstHistory + from ._preferential_history import History + Attribute = TypedDict( "Attribute", { @@ -128,6 +133,7 @@ def serialize_study_detail( union: list[tuple[str, BaseDistribution]], union_user_attrs: list[tuple[str, bool]], has_intermediate_values: bool, + plotly_graph_objects: dict[str, str], ) -> dict[str, Any]: serialized: dict[str, Any] = { "name": summary.study_name, @@ -157,10 +163,38 @@ def serialize_study_detail( if form_widgets: serialized["form_widgets"] = form_widgets if serialized["is_preferential"]: - serialized["preference_history"] = serialize_preference_histories(system_attrs) + serialized["preference_history"] = serialize_preference_history(system_attrs) + serialized["plotly_graph_objects"] = [ + {"id": id_, "graph_object": graph_object} + for id_, graph_object in plotly_graph_objects.items() + ] return serialized +def serialize_preference_history( + system_attrs: dict[str, Any], +) -> list[History]: + histories: list[History] = [] + for k, v in system_attrs.items(): + if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): + continue + choice: dict[str, Any] = json.loads(v) + if choice["mode"] == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": choice["id"], + "preference_id": choice["preference_id"], + "timestamp": choice["timestamp"], + "candidates": choice["candidates"], + "clicked": choice["clicked"], + "enabled": is_preference_valid(choice["preference_id"]), + } + histories.append(history) + + histories.sort(key=lambda c: datetime.fromisoformat(c["timestamp"])) + return histories + + def serialize_frozen_trial( study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any] ) -> dict[str, Any]: diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index fba9b4cd..6e093683 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -12,14 +12,16 @@ from optuna.samplers import BaseSampler from optuna.samplers import RandomSampler from optuna.trial import FrozenTrial from optuna.trial import TrialState +from optuna_dashboard.preferential._system_attrs import get_n_generate from optuna_dashboard.preferential._system_attrs import get_preferences +from optuna_dashboard.preferential._system_attrs import get_skipped_trial_ids from optuna_dashboard.preferential._system_attrs import is_skipped_trial from optuna_dashboard.preferential._system_attrs import report_preferences +from optuna_dashboard.preferential._system_attrs import set_n_generate _logger = logging.get_logger(__name__) _SYSTEM_ATTR_PREFERENTIAL_STUDY = "preference:is_preferential" -_SYSTEM_ATTR_COMPARISON_READY = "preference:comparison_ready" class PreferentialStudy: @@ -60,13 +62,6 @@ class PreferentialStudy: def best_trials(self) -> list[FrozenTrial]: """Return the trials that is not dominated by other trials. - .. seealso:: - - See `Study.best_trials`_ for details. - - .. _Study.best_trials: https://optuna.readthedocs.io/en/stable/reference/\ - generated/optuna.study.Study.html#optuna.study.Study.best_trials - Returns: A list of FrozenTrial object """ @@ -180,6 +175,38 @@ class PreferentialStudy: """ self._study.add_trials(trials) + def enqueue_trial( + self, + params: dict[str, Any], + user_attrs: dict[str, Any] | None = None, + skip_if_exists: bool = False, + ) -> None: + """Enqueue a trial with given parameter values. + + You can fix the next sampling parameters which will be evaluated in your + objective function. + + .. seealso:: + + See `Study.enqueue_trials`_ for details. + + .. _Study.get_trials: https://optuna.readthedocs.io/en/stable/reference/\ + generated/optuna.study.Study.html#optuna.study.Study.enqueue_trials + + Args: + params: + Parameter values to pass your objective function. + user_attrs: + A dictionary of user-specific attributes other than ``params``. + skip_if_exists: + When :obj:`True`, prevents duplicate trials from being enqueued again. + + .. note:: + This method might produce duplicated trials if called simultaneously + by multiple processes at the same time with same ``params`` dict. + """ + self._study.enqueue_trial(params, user_attrs, skip_if_exists) + def report_preference( self, better_trials: FrozenTrial | list[FrozenTrial], @@ -217,8 +244,11 @@ class PreferentialStudy: Returns: A list of the pair of FrozenTrial objects. The left trial is better than the right one. """ + + preferences = get_preferences( + self._study._storage.get_study_system_attrs(self._study._study_id) + ) # Must come before study.get_trials() trials = self._study.get_trials(deepcopy=deepcopy) - preferences = get_preferences(self._study._study_id, self._study._storage) return [(trials[better], trials[worse]) for (better, worse) in preferences] def set_user_attr(self, key: str, value: Any) -> None: @@ -235,37 +265,41 @@ class PreferentialStudy: """ self._study.set_user_attr(key, value) - def mark_comparison_ready(self, trial_or_number: optuna.Trial | int) -> None: - """Mark trials ready to compare. + def should_generate(self) -> bool: + """Return whether the generator should generate a new trial now. - Args: - trial_or_number: - A Trial object or trial_number. + Returns :obj:`True` if the number of trials not reported bad and not skipped are less than + :attr:`~optuna_dashboard.preferential.PreferentialStudy.n_generate`. Users are recommended + to generate a new trial if this method returns :obj:`True`, and to wait for human + evaluation if this method returns :obj:`False`. """ - storage = self._study._storage - if isinstance(trial_or_number, optuna.Trial): - trial_id = trial_or_number._trial_id - elif isinstance(trial_or_number, int): - trial_id = storage.get_trial_id_from_study_id_trial_number( - self._study._study_id, trial_or_number - ) - else: - raise RuntimeError("Unexpected trial type") - storage.set_trial_system_attr(trial_id, _SYSTEM_ATTR_COMPARISON_READY, True) + study_system_attrs = self._study._storage.get_study_system_attrs( + self._study._study_id + ) # Must come before _study.get_trials() + trials = self._study.get_trials( + deepcopy=False, states=(TrialState.COMPLETE, TrialState.RUNNING) + ) + worse_trial_numbers = {worse for _, worse in get_preferences(study_system_attrs)} + skipped_trial_ids = set(get_skipped_trial_ids(study_system_attrs)) + active_trials = [ + t + for t in trials + if t.number not in worse_trial_numbers and t._trial_id not in skipped_trial_ids + ] + return len(active_trials) < get_n_generate(self._study.system_attrs) def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]: - preferences = get_preferences(study_id, storage) + preferences = get_preferences(storage.get_study_system_attrs(study_id)) worse_numbers = {worse for _, worse in preferences} + nondominated_numbers = {better for better, _ in preferences if better not in worse_numbers} + trials = storage.get_all_trials(study_id, deepcopy=False) + study_system_attrs = storage.get_study_system_attrs(study_id) + best_trials = [] - for t in storage.get_all_trials( - study_id, deepcopy=False, states=(TrialState.COMPLETE, TrialState.RUNNING) - ): - if not t.system_attrs.get(_SYSTEM_ATTR_COMPARISON_READY, False): - continue - if t.number in worse_numbers: - continue + for n in nondominated_numbers: + t = trials[n] if is_skipped_trial(t._trial_id, study_system_attrs): continue best_trials.append(copy.deepcopy(t)) @@ -274,6 +308,7 @@ def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list def create_study( *, + n_generate: int, storage: str | optuna.storages.BaseStorage | None = None, sampler: BaseSampler | None = None, study_name: str | None = None, @@ -293,6 +328,12 @@ def create_study( trial = study.ask() Args: + n_generate: + The number of active trials to keep. + :func:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns + :obj:`True` if the number of trials not reported bad and not skipped are less than + ``n_generate``. + storage: Database URL. If this argument is set to None, in-memory storage is used, and the :class:`~optuna_dashboard.preferential.PreferentialStudy` will not be persistent. @@ -328,6 +369,7 @@ def create_study( study._storage.set_study_system_attr( study._study_id, _SYSTEM_ATTR_PREFERENTIAL_STUDY, True ) + set_n_generate(study._study_id, study._storage, n_generate) return PreferentialStudy(study) except optuna.exceptions.DuplicatedStudyError: diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 4e3f55f5..0c49a8a8 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -9,6 +9,7 @@ from optuna.trial import TrialState _SYSTEM_ATTR_PREFIX_PREFERENCE = "preference:values" _SYSTEM_ATTR_PREFIX_SKIP_TRIAL = "preference:skip_trial:" +_SYSTEM_ATTR_N_GENERATE = "preference:n_generate" def report_preferences( @@ -16,8 +17,8 @@ def report_preferences( storage: BaseStorage, preferences: list[tuple[int, int]], ) -> str: - preference_uuid = str(uuid.uuid4()) - key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_uuid + preference_id = str(uuid.uuid4()) + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_id storage.set_study_system_attr( study_id=study_id, key=key, @@ -31,27 +32,24 @@ def report_preferences( trial_id = trials[number]._trial_id if trials[number].state != TrialState.COMPLETE: storage.set_trial_state_values(trial_id, TrialState.COMPLETE, values) - return preference_uuid + return preference_id -def get_preference(study_id: int, storage: BaseStorage, uuid: str) -> list[tuple[int, int]]: - system_attrs = storage.get_study_system_attrs(study_id) - return system_attrs.get(_SYSTEM_ATTR_PREFIX_PREFERENCE + uuid, []) # type: ignore - - -def get_preferences( - study_id: int, - storage: BaseStorage, -) -> list[tuple[int, int]]: +def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]]: preferences: list[tuple[int, int]] = [] - system_attrs = storage.get_study_system_attrs(study_id) - for k, v in system_attrs.items(): + for k, v in study_system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_PREFERENCE): continue preferences.extend(v) # type: ignore return preferences +def is_preference_valid(study_system_attrs: dict[str, Any], uuid: str) -> bool: + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + uuid + preference = study_system_attrs.get(key, []) + return len(preference) == 0 + + def report_skip( study_id: int, trial_id: int, @@ -67,3 +65,28 @@ def report_skip( def is_skipped_trial(trial_id: int, study_system_attrs: dict[str, Any]) -> bool: key = _SYSTEM_ATTR_PREFIX_SKIP_TRIAL + str(trial_id) return key in study_system_attrs + + +def get_skipped_trial_ids(study_system_attrs: dict[str, Any]) -> list[int]: + skipped_trial_ids: list[int] = [] + for k in study_system_attrs: + if not k.startswith(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL): + continue + try: + trial_id = int(k[len(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL) :]) # noqa: E203 + skipped_trial_ids.append(trial_id) + except ValueError: + continue + return skipped_trial_ids + + +def get_n_generate(study_system_attrs: dict[str, Any]) -> int: + return study_system_attrs[_SYSTEM_ATTR_N_GENERATE] + + +def set_n_generate(study_id: int, storage: BaseStorage, n_generate: int) -> None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_N_GENERATE, + value=n_generate, + ) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index ff4001c1..b90a1de4 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -342,7 +342,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): if len(search_space) == 0: return {} - preferences = get_preferences(study._study_id, study._storage) + preferences = get_preferences(study.system_attrs) trials = study.get_trials(deepcopy=False) if len(preferences) == 0: return {} diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 6886f8b4..5c9355b7 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -55,8 +55,9 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } -interface PreferenceChoiceResponce { - uuid: string +interface PreferenceHistoryResponce { + id: string + preference_id: string candidates: number[] clicked: number mode: PreferenceFeedbackMode @@ -64,11 +65,12 @@ interface PreferenceChoiceResponce { enabled: boolean } -const convertPreferenceChoice = ( - res: PreferenceChoiceResponce -): PreferenceChoice => { +const convertPreferenceHistory = ( + res: PreferenceHistoryResponce +): PreferenceHistory => { return { - uuid: res.uuid, + id: res.id, + preference_id: res.preference_id, candidates: res.candidates, clicked: res.clicked, feedback_mode: res.mode, @@ -92,7 +94,8 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets - preference_history?: PreferenceChoiceResponce[] + preference_history?: PreferenceHistoryResponce[] + plotly_graph_objects: PlotlyGraphObject[] } export const getStudyDetailAPI = ( @@ -129,8 +132,9 @@ export const getStudyDetailAPI = ( form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, preference_history: res.data.preference_history?.map( - convertPreferenceChoice + convertPreferenceHistory ), + plotly_graph_objects: res.data.plotly_graph_objects, } }) } diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 184ec606..446b362e 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -221,7 +221,7 @@ export const AppDrawer: FC<{ component={Link} to={`${URL_PREFIX}/studies/${studyId}/preference-history`} sx={styleListItemButton} - selected={page === "analytics"} + selected={page === "preferenceHistory"} > diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 76725faa..c9444237 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -13,10 +13,11 @@ import UndoIcon from "@mui/icons-material/Undo" import RedoIcon from "@mui/icons-material/Redo" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import Modal from "@mui/material/Modal" +import { red } from "@mui/material/colors" import { TrialListDetail } from "./TrialList" import { MarkdownRenderer } from "./Note" -import { red } from "@mui/material/colors" +import { formatDate } from "../dateUtil" import { actionCreator } from "../action" type TrialType = "worst" | "none" @@ -26,7 +27,7 @@ const CandidateTrial: FC<{ type: TrialType }> = ({ trial, type }) => { const theme = useTheme() - const trialWidth = 500 + const trialWidth = 300 const trialHeight = 300 const [detailShown, setDetailShown] = useState(false) @@ -137,7 +138,7 @@ const CandidateTrial: FC<{ } const ChoiceTrials: FC<{ - choice: PreferenceChoice + choice: PreferenceHistory trials: Trial[] study_id: number }> = ({ choice, trials, study_id }) => { @@ -145,47 +146,31 @@ const ChoiceTrials: FC<{ const worst_trials = new Set([choice.clicked]) const actions = actionCreator() const handleUndo = () => { - actions.switchPreferentialHistory(study_id, choice.uuid, false) + actions.switchPreferentialHistory(study_id, choice.id, false) } const handleRedo = () => { - actions.switchPreferentialHistory(study_id, choice.uuid, true) + actions.switchPreferentialHistory(study_id, choice.id, true) } return ( - - + - - {choice.timestamp.toLocaleString()} - - - - - - - - + {formatDate(choice.timestamp)} + {choice.candidates.map((trial_num, index) => ( @@ -211,8 +196,9 @@ export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({ return null } const theme = useTheme() + const preference_histories = [...studyDetail.preference_history] - if (studyDetail.preference_history.length === 0) { + if (preference_histories.length === 0) { return ( = ({ padding={theme.spacing(2)} sx={{ display: "flex", flexDirection: "column" }} > - {studyDetail.preference_history.map((choice) => ( + {preference_histories.reverse().map((choice) => ( true} + isBestTrial={() => isBestTrial} directions={[]} objectiveNames={[]} /> @@ -182,11 +184,15 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ return null } const theme = useTheme() + + const runningTrials = studyDetail.trials.filter((t) => t.state === "Running") + const activeTrials = runningTrials.concat(studyDetail.best_trials) + const [displayTrials, setDisplayTrials] = useState({ - numbers: studyDetail.best_trials.map((t) => t.number), - last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1), + numbers: activeTrials.map((t) => t.number), + last_number: Math.max(...activeTrials.map((t) => t.number), -1), }) - const new_trails = studyDetail.best_trials.filter( + const new_trails = activeTrials.filter( (t) => displayTrials.last_number < t.number && displayTrials.numbers.find((n) => n === t.number) === undefined @@ -239,7 +245,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {displayTrials.numbers.map((t, index) => ( trial.number === t)} + trial={activeTrials.find((trial) => trial.number === t)} candidates={displayTrials.numbers.filter((n) => n !== -1)} hideTrial={() => { hideTrial(t) diff --git a/optuna_dashboard/ts/components/StudyHistory.tsx b/optuna_dashboard/ts/components/StudyHistory.tsx index b47c557a..907acd1a 100644 --- a/optuna_dashboard/ts/components/StudyHistory.tsx +++ b/optuna_dashboard/ts/components/StudyHistory.tsx @@ -15,6 +15,7 @@ import { GraphIntermediateValues } from "./GraphIntermediateValues" import Grid2 from "@mui/material/Unstable_Grid2" import { DataGrid, DataGridColumn } from "./DataGrid" import { GraphHyperparameterImportance } from "./GraphHyperparameterImportances" +import { UserDefinedPlot } from "./UserDefinedPlot" import { BestTrialsCard } from "./BestTrialsCard" import { useStudyDetailValue, @@ -124,6 +125,16 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => { + {studyDetail !== null && + studyDetail.plotly_graph_objects.map((go) => ( + + + + + + + + ))} diff --git a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx index 536f1d91..ec30e63f 100644 --- a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx +++ b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx @@ -1,10 +1,11 @@ import * as THREE from "three" -import React, { useEffect, useState } from "react" +import React, { useEffect, useState, ReactNode } from "react" import { Canvas } from "@react-three/fiber" import { GizmoHelper, GizmoViewport, OrbitControls } from "@react-three/drei" import { STLLoader } from "three/examples/jsm/loaders/STLLoader" import { Rhino3dmLoader } from "three/examples/jsm/loaders/3DMLoader" import { PerspectiveCamera } from "three" +import { Modal, Box } from "@mui/material" interface ThreejsArtifactViewerProps { src: string @@ -109,3 +110,48 @@ export const ThreejsArtifactViewer: React.FC = ( ) } + +export const useThreejsArtifactModal = (): [ + (path: string, artifact: Artifact) => void, + () => ReactNode +] => { + const [open, setOpen] = useState(false) + const [target, setTarget] = useState<[string, Artifact | null]>(["", null]) + + const openModal = (artifactUrlPath: string, artifact: Artifact) => { + setTarget([artifactUrlPath, artifact]) + setOpen(true) + } + + const renderDeleteStudyDialog = () => { + return ( + { + setOpen(false) + setTarget(["", null]) + }} + > + + + + + ) + } + return [openModal, renderDeleteStudyDialog] +} diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 4aceb619..84f98c0c 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -20,7 +20,6 @@ import { CardContent, CardMedia, CardActionArea, - Modal, } from "@mui/material" import Chip from "@mui/material/Chip" import Divider from "@mui/material/Divider" @@ -47,7 +46,10 @@ import { artifactIsAvailable } from "../state" import { actionCreator } from "../action" import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" import { TrialFormWidgets } from "./TrialFormWidgets" -import { ThreejsArtifactViewer } from "./ThreejsArtifactViewer" +import { + ThreejsArtifactViewer, + useThreejsArtifactModal, +} from "./ThreejsArtifactViewer" const states: TrialState[] = [ "Complete", @@ -330,9 +332,8 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = useDeleteArtifactDialog() const [dragOver, setDragOver] = useState(false) - const [open3dModelViewer, setOpen3dModelViewer] = useState<{ - [key: string]: boolean - }>({}) + const [openThreejsArtifactModal, renderThreejsArtifactModal] = + useThreejsArtifactModal() const width = "200px" const height = "150px" @@ -499,48 +500,12 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { color="inherit" sx={{ margin: "auto 0" }} onClick={() => { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = true - return obj - }) + const urlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}` + openThreejsArtifactModal(urlPath, a) }} > - { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = false - return obj - }) - }} - > - - - - = ({ trial }) => { ) : null} {renderDeleteArtifactDialog()} + {renderThreejsArtifactModal()} ) } diff --git a/optuna_dashboard/ts/components/UserDefinedPlot.tsx b/optuna_dashboard/ts/components/UserDefinedPlot.tsx new file mode 100644 index 00000000..029c4b57 --- /dev/null +++ b/optuna_dashboard/ts/components/UserDefinedPlot.tsx @@ -0,0 +1,21 @@ +import * as plotly from "plotly.js-dist-min" +import React, { FC, useEffect } from "react" +import { Box } from "@mui/material" + +export const UserDefinedPlot: FC<{ + graphObject: PlotlyGraphObject +}> = ({ graphObject }) => { + const plotDomId = `user-defined-plot:${graphObject.id}` + + useEffect(() => { + try { + const parsed = JSON.parse(graphObject.graph_object) + plotly.react(plotDomId, parsed.data, parsed.layout) + } catch (e) { + // Avoid to crash the whole page when given invalid grpah objects. + console.error(e) + } + }, [graphObject]) + + return +} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 26c08d38..fd9ca113 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -182,6 +182,11 @@ type FormWidgets = widgets: UserAttrFormWidget[] } +type PlotlyGraphObject = { + id: string + graph_object: string +} + type StudyDetail = { id: number name: string @@ -198,7 +203,8 @@ type StudyDetail = { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets - preference_history?: PreferenceChoice[] + preference_history?: PreferenceHistory[] + plotly_graph_objects: PlotlyGraphObject[] } type StudyDetails = { @@ -209,8 +215,9 @@ type StudyParamImportance = { [study_id: string]: ParamImportance[][] } -type PreferenceChoice = { - uuid: string +type PreferenceHistory = { + id: string + preference_id: string candidates: number[] clicked: number feedback_mode: PreferenceFeedbackMode diff --git a/pyproject.toml b/pyproject.toml index 0555f701..9b7ce731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ docs = [ test = [ "coverage", + "plotly", "pytest", "moto[s3]", ] diff --git a/python_tests/preferential/test_study.py b/python_tests/preferential/test_study.py index 3e0b011c..7fc1aaba 100644 --- a/python_tests/preferential/test_study.py +++ b/python_tests/preferential/test_study.py @@ -25,7 +25,7 @@ from ..storage_supplier import StorageSupplier @parametrize_storages def test_study_set_and_get_user_attrs(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) study.set_user_attr("dataset", "MNIST") assert study.user_attrs["dataset"] == "MNIST" @@ -34,13 +34,12 @@ def test_study_set_and_get_user_attrs(storage_supplier: Callable[[], StorageSupp @parametrize_storages def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) assert len(study.preferences) == 0 for _ in range(2): trial = study.ask() trial.suggest_float("x", 0, 1) - study.mark_comparison_ready(trial) better, worse = study.trials study.report_preference(better, worse) assert len(study.preferences) == 1 @@ -51,7 +50,9 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli def test_study_pickle() -> None: - study_1 = create_study() + study_1 = create_study( + n_generate=4, + ) for _ in range(10): study_1.ask() assert len(study_1.trials) == 10 @@ -69,13 +70,17 @@ def test_study_pickle() -> None: def test_create_study(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: # Test creating a new study. - study = create_study(storage=storage, load_if_exists=False) + study = create_study(n_generate=4, storage=storage, load_if_exists=False) # Test `load_if_exists=True` with existing study. - create_study(study_name=study.study_name, storage=storage, load_if_exists=True) + create_study( + n_generate=4, study_name=study.study_name, storage=storage, load_if_exists=True + ) with pytest.raises(DuplicatedStudyError): - create_study(study_name=study.study_name, storage=storage, load_if_exists=False) + create_study( + n_generate=4, study_name=study.study_name, storage=storage, load_if_exists=False + ) @parametrize_storages @@ -92,7 +97,7 @@ def test_load_study(storage_supplier: Callable[[], StorageSupplier]) -> None: load_study(study_name=study_name, storage=storage) # Create a new study. - created_study = create_study(study_name=study_name, storage=storage) + created_study = create_study(n_generate=4, study_name=study_name, storage=storage) # Test loading an existing study. loaded_study = load_study(study_name=study_name, storage=storage) @@ -108,7 +113,7 @@ def test_load_study_study_name_none(storage_supplier: Callable[[], StorageSuppli study_name = str(uuid.uuid4()) - _ = create_study(study_name=study_name, storage=storage) + _ = create_study(n_generate=4, study_name=study_name, storage=storage) loaded_study = load_study(study_name=None, storage=storage) @@ -116,7 +121,7 @@ def test_load_study_study_name_none(storage_supplier: Callable[[], StorageSuppli study_name = str(uuid.uuid4()) - _ = create_study(study_name=study_name, storage=storage) + _ = create_study(n_generate=4, study_name=study_name, storage=storage) # Ambiguous study. with pytest.raises(ValueError): @@ -131,7 +136,7 @@ def test_delete_study(storage_supplier: Callable[[], StorageSupplier]) -> None: delete_study(study_name="invalid-study-name", storage=storage) # Test deleting an existing study. - study = create_study(storage=storage, load_if_exists=False) + study = create_study(n_generate=4, storage=storage, load_if_exists=False) delete_study(study_name=study.study_name, storage=storage) # Test failed to delete the study which is already deleted. @@ -141,12 +146,11 @@ def test_delete_study(storage_supplier: Callable[[], StorageSupplier]) -> None: def test_copy_study() -> None: with StorageSupplier("sqlite") as from_storage, StorageSupplier("sqlite") as to_storage: - from_study = create_study(storage=from_storage) + from_study = create_study(n_generate=4, storage=from_storage) from_study.set_user_attr("baz", "qux") for _ in range(3): trial = from_study.ask() trial.suggest_float("x", 0, 1) - from_study.mark_comparison_ready(trial) from_study.report_preference(from_study.trials[0], from_study.trials[1]) from_study.report_preference(from_study.trials[1], from_study.trials[2]) @@ -165,8 +169,8 @@ def test_copy_study() -> None: def test_copy_study_to_study_name() -> None: with StorageSupplier("sqlite") as from_storage, StorageSupplier("sqlite") as to_storage: - from_study = create_study(study_name="foo", storage=from_storage) - _ = create_study(study_name="foo", storage=to_storage) + from_study = create_study(n_generate=4, study_name="foo", storage=from_storage) + _ = create_study(n_generate=4, study_name="foo", storage=to_storage) with pytest.raises(DuplicatedStudyError): copy_study( @@ -188,7 +192,7 @@ def test_copy_study_to_study_name() -> None: @parametrize_storages def test_add_trial(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) assert len(study.trials) == 0 trial = create_trial(value=0) @@ -198,7 +202,9 @@ def test_add_trial(storage_supplier: Callable[[], StorageSupplier]) -> None: def test_add_trial_invalid_values_length() -> None: - study = create_study() + study = create_study( + n_generate=4, + ) trial = create_trial(values=[0, 0]) with pytest.raises(ValueError): study.add_trial(trial) @@ -207,7 +213,7 @@ def test_add_trial_invalid_values_length() -> None: @parametrize_storages def test_add_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) assert len(study.trials) == 0 study.add_trials([]) @@ -220,7 +226,7 @@ def test_add_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: assert trial.number == i assert trial.value == i - other_study = create_study(storage=storage) + other_study = create_study(n_generate=4, storage=storage) other_study.add_trials(study.trials) assert len(other_study.trials) == 3 for i, trial in enumerate(other_study.trials): @@ -231,11 +237,10 @@ def test_add_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: @parametrize_storages def test_get_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) for _ in range(5): trial = study.ask() trial.suggest_int("x", 1, 5) - study.mark_comparison_ready(trial) with patch("copy.deepcopy", wraps=copy.deepcopy) as mock_object: trials0 = study.get_trials(deepcopy=False) @@ -256,10 +261,9 @@ def test_get_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: @parametrize_storages def test_get_trials_state_option(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() better, worse = study.trials[:2] study.report_preference(better, worse) @@ -286,7 +290,9 @@ def test_get_trials_state_option(storage_supplier: Callable[[], StorageSupplier] def test_ask() -> None: - study = create_study() + study = create_study( + n_generate=4, + ) trial = study.ask() assert isinstance(trial, Trial) @@ -298,7 +304,9 @@ def test_ask_fixed_search_space() -> None: "y": distributions.CategoricalDistribution(["bacon", "spam"]), } - study = create_study() + study = create_study( + n_generate=4, + ) trial = study.ask(fixed_distributions=fixed_distributions) params = trial.params @@ -312,7 +320,7 @@ def test_report_preferences_from_another_process() -> None: with StorageSupplier("sqlite") as storage: # Create a study and ask for a new trial. - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) study.ask() study.ask() diff --git a/python_tests/preferential/test_system_attrs.py b/python_tests/preferential/test_system_attrs.py index 10448d48..34f93200 100644 --- a/python_tests/preferential/test_system_attrs.py +++ b/python_tests/preferential/test_system_attrs.py @@ -18,12 +18,13 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli study.ask() study_id = study._study_id - assert len(get_preferences(study_id, storage)) == 0 + + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 0 better, worse = study.trials[0], study.trials[1] report_preferences(study_id, storage, [(better.number, worse.number)]) - assert len(get_preferences(study_id, storage)) == 1 + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 1 - actual_better, actual_worse = get_preferences(study_id, storage)[0] + actual_better, actual_worse = get_preferences(storage.get_study_system_attrs(study_id))[0] assert actual_better == better.number assert actual_worse == worse.number diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 1fc92508..61913e20 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -103,12 +103,13 @@ class APITestCase(TestCase): def test_get_best_trials_of_preferential_study(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() study.report_preference(study.trials[0], study.trials[1]) + assert len(study.best_trials) == 1 + app = create_app(storage) study_id = study._study._study_id status, _, body = send_request( @@ -120,16 +121,14 @@ class APITestCase(TestCase): self.assertEqual(status, 200) best_trials = json.loads(body)["best_trials"] - assert len(best_trials) == 2 + assert len(best_trials) == 1 assert best_trials[0]["number"] == 0 - assert best_trials[1]["number"] == 2 def test_report_preference(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id @@ -158,29 +157,52 @@ class APITestCase(TestCase): assert better.number == 2 assert worse.number == 1 - def test_skip_trial(self) -> None: + def test_report_preference_when_typo_mode(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) - trials: list[optuna.Trial] = [] + study = create_study(storage=storage, n_generate=3) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) - trials.append(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id status, _, _ = send_request( app, - f"/api/studies/{study_id}/{trials[1]._trial_id}/skip", + f"/api/studies/{study_id}/preference", + "POST", + body=json.dumps( + { + "mode": "ChoseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 400) + + def test_skip_trial(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(n_generate=4, storage=storage) + trials: list[optuna.Trial] = [] + for _ in range(3): + trial = study.ask() + trials.append(trial) + study.report_preference(study.trials[0], study.trials[1]) + study.report_preference(study.trials[2], study.trials[1]) + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/{trials[0]._trial_id}/skip", "POST", content_type="application/json", ) self.assertEqual(status, 204) best_trials = study.best_trials - assert len(best_trials) == 2 - assert best_trials[0].number == 0 - assert best_trials[1].number == 2 + assert len(best_trials) == 1 + assert best_trials[0].number == 2 def test_undo_redo_history(self) -> None: storage = optuna.storages.InMemoryStorage() diff --git a/python_tests/test_custom_plot_data.py b/python_tests/test_custom_plot_data.py new file mode 100644 index 00000000..3dcfc856 --- /dev/null +++ b/python_tests/test_custom_plot_data.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import optuna +from optuna_dashboard import _custom_plot_data as custom_plot_data +from optuna_dashboard import save_plotly_graph_object +import pytest + + +def get_dummy_study() -> optuna.Study: + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + optuna.logging.set_verbosity(optuna.logging.ERROR) + study.optimize(objective, n_trials=100) + return study + + +def test_save_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 2 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +def test_update_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object( + dummy_study, plot_data, graph_object_id=graph_object_id + ) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +@pytest.mark.parametrize( + "name", + [ + "0", + "a", + "a1-:_.", + ], +) +def test_is_valid_graph_object_id(name: str) -> None: + assert custom_plot_data.is_valid_graph_object_id(name) + + +@pytest.mark.parametrize( + "name", + [ + "a,", + "a b", + "aあいうえお", + ], +) +def test_is_invalid_graph_object_id(name: str) -> None: + assert not custom_plot_data.is_valid_graph_object_id(name) diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 7b288968..eb5ffbce 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -2,13 +2,11 @@ from __future__ import annotations from typing import Callable -from optuna_dashboard._preferential_history import load_preference_history +from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import report_history -from optuna_dashboard._preferential_history import switching_history -from optuna_dashboard._serializer import serialize_preference_histories +from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE -from optuna_dashboard.preferential._system_attrs import get_preference from .storage_supplier import parametrize_storages from .storage_supplier import StorageSupplier @@ -17,38 +15,37 @@ from .storage_supplier import StorageSupplier @parametrize_storages def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: - study = create_study(storage=storage) + study = create_study(storage=storage, n_generate=5) for _ in range(5): trial = study.ask() trial.suggest_float("x", 0, 1) - study.mark_comparison_ready(trial) study_id = study._study._study_id report_history( study_id=study_id, storage=storage, - input_data={ - "mode": "ChooseWorst", - "candidates": [0, 1, 2], - "clicked": 1, - }, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 1, 2], + clicked=1, + ), ) report_history( study_id=study_id, storage=storage, - input_data={ - "mode": "ChooseWorst", - "candidates": [0, 2, 3, 4], - "clicked": 0, - }, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 2, 3, 4], + clicked=0, + ), ) - history = serialize_preference_histories(storage.get_study_system_attrs(study_id)) + history = serialize_preference_history(storage.get_study_system_attrs(study_id)) sys_attrs = storage.get_study_system_attrs(study_id) assert len(history) == 2 assert history[0]["candidates"] == [0, 1, 2] assert history[0]["clicked"] == 1 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_uuid"]] + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]] assert len(preferences) == 2 for i, (best, worst) in enumerate([(0, 1), (2, 1)]): assert len(preferences[i]) == 2 @@ -56,7 +53,7 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert preferences[i][1] == worst assert history[1]["candidates"] == [0, 2, 3, 4] assert history[1]["clicked"] == 0 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_uuid"]] + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]] assert len(preferences) == 3 for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): assert len(preferences[i]) == 2 @@ -64,56 +61,56 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert preferences[i][1] == worst -@parametrize_storages -def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> None: - with storage_supplier() as storage: - study = create_study(storage=storage) - for _ in range(5): - trial = study.ask() - trial.suggest_float("x", 0, 1) - study.mark_comparison_ready(trial) +# TODO(moririn): Add tests for switching_history. +# @parametrize_storages +# def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> None: +# with storage_supplier() as storage: +# study = create_study(storage=storage, n_generate=5) +# for _ in range(5): +# trial = study.ask() +# trial.suggest_float("x", 0, 1) - study_id = study._study._study_id +# study_id = study._study._study_id - history_uuid = report_history( - study_id=study_id, - storage=storage, - input_data={ - "mode": "ChooseWorst", - "candidates": [0, 1, 2], - "clicked": 1, - }, - ) - switching_history(study_id, storage, history_uuid, False) - history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) - preference = get_preference(study_id, storage, history.preference_uuid) - assert history.mode == "ChooseWorst" - assert history.candidates == [0, 1, 2] - assert history.clicked == 1 - assert len(history.evacuated_preference) == 2 - assert len(preference) == 0 +# history_uuid = report_history( +# study_id=study_id, +# storage=storage, +# input_data={ +# "mode": "ChooseWorst", +# "candidates": [0, 1, 2], +# "clicked": 1, +# }, +# ) +# switching_history(study_id, storage, history_uuid, False) +# history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) +# preference = get_preference(study_id, storage, history.preference_uuid) +# assert history.mode == "ChooseWorst" +# assert history.candidates == [0, 1, 2] +# assert history.clicked == 1 +# assert len(history.evacuated_preference) == 2 +# assert len(preference) == 0 - switching_history(study_id, storage, history_uuid, False) - history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) - preference = get_preference(study_id, storage, history.preference_uuid) - assert len(history.evacuated_preference) == 2 - assert len(preference) == 0 +# switching_history(study_id, storage, history_uuid, False) +# history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) +# preference = get_preference(study_id, storage, history.preference_uuid) +# assert len(history.evacuated_preference) == 2 +# assert len(preference) == 0 - switching_history(study_id, storage, history_uuid, True) - history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) - preference = get_preference(study_id, storage, history.preference_uuid) - assert history.mode == "ChooseWorst" - assert history.candidates == [0, 1, 2] - assert history.clicked == 1 - assert len(history.evacuated_preference) == 0 - assert len(preference) == 2 - for i, (best, worst) in enumerate([(0, 1), (2, 1)]): - assert len(preference[i]) == 2 - assert preference[i][0] == best - assert preference[i][1] == worst +# switching_history(study_id, storage, history_uuid, True) +# history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) +# preference = get_preference(study_id, storage, history.preference_uuid) +# assert history.mode == "ChooseWorst" +# assert history.candidates == [0, 1, 2] +# assert history.clicked == 1 +# assert len(history.evacuated_preference) == 0 +# assert len(preference) == 2 +# for i, (best, worst) in enumerate([(0, 1), (2, 1)]): +# assert len(preference[i]) == 2 +# assert preference[i][0] == best +# assert preference[i][1] == worst - switching_history(study_id, storage, history_uuid, True) - history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) - preference = get_preference(study_id, storage, history.preference_uuid) - assert len(history.evacuated_preference) == 0 - assert len(preference) == 2 +# switching_history(study_id, storage, history_uuid, True) +# history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) +# preference = get_preference(study_id, storage, history.preference_uuid) +# assert len(history.evacuated_preference) == 0 +# assert len(preference) == 2 diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index a75db32d..72db7b26 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -24,12 +24,12 @@ def test_serialize_dict() -> None: def test_get_study_detail_is_preferential() -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(n_generate=4, storage=storage) study_summaries = get_study_summaries(storage) assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert study_detail["is_preferential"] @@ -40,13 +40,13 @@ def test_get_study_detail_is_not_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert not study_detail["is_preferential"] def test_get_study_summary_is_preferential() -> None: storage = optuna.storages.InMemoryStorage() - create_study(storage=storage) + create_study(n_generate=4, storage=storage) study_summaries = get_study_summaries(storage) assert len(study_summaries) == 1