From 0fc9f0126c90f8c8550f9e3a68bec44a9c28b094 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 11 Jun 2023 17:39:16 +0900 Subject: [PATCH 01/12] Setup the server and browser of Optuna Dashboard in e2e tests --- e2e_tests/conftest.py | 207 ++++++++++++++++++++++++++++ e2e_tests/visual_regression_test.py | 34 ++--- 2 files changed, 224 insertions(+), 17 deletions(-) create mode 100644 e2e_tests/conftest.py diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py new file mode 100644 index 00000000..58338334 --- /dev/null +++ b/e2e_tests/conftest.py @@ -0,0 +1,207 @@ +import pytest +from playwright.sync_api import sync_playwright + +import threading +from wsgiref.simple_server import make_server + +import optuna +from optuna_dashboard import wsgi + + +@pytest.fixture(scope="session") +def dummy_storage() -> optuna.storages.InMemoryStorage: + + storage = optuna.storages.InMemoryStorage() + sampler = optuna.samplers.RandomSampler(seed=0) + + # Single-objective study + study = optuna.create_study(study_name="single", storage=storage, sampler=sampler) + + def objective_single(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective_single, n_trials=50) + + # A single objective study with a single trial + # Refs: https://github.com/optuna/optuna-dashboard/issues/401 + study = optuna.create_study(study_name="single-trial", storage=storage, sampler=sampler) + study.optimize(objective_single, n_trials=1) + + # Single-objective study with 1 parameter + study = optuna.create_study( + study_name="single-1-param", storage=storage, direction="maximize", sampler=sampler + ) + + def objective_single_with_1param(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + return -((x1 - 2) ** 2) + + study.optimize(objective_single_with_1param, n_trials=50) + + # Single-objective study with dynamic search space + study = optuna.create_study( + study_name="single-dynamic", storage=storage, direction="maximize", sampler=sampler + ) + + def objective_single_dynamic(trial: optuna.Trial) -> float: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + return (trial.suggest_float("x1", 0, 10) - 2) ** 2 + else: + return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) + + study.optimize(objective_single_dynamic, n_trials=50) + + # Single objective study with 'inf', '-inf', or 'nan' value + study = optuna.create_study(study_name="single-inf", storage=storage, sampler=sampler) + + def objective_single_inf(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + return float("inf") + elif trial.number % 3 == 1: + return float("-inf") + else: + return x**2 + + study.optimize(objective_single_inf, n_trials=50) + + # Multi-objective study + study = optuna.create_study( + study_name="multi-objective", + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective_multi(trial: optuna.Trial) -> tuple[float, float]: + x = trial.suggest_float("x", 0, 5) + y = trial.suggest_float("y", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + + study.optimize(objective_multi, n_trials=50) + + # Multi-objective study with dynamic search space + study = optuna.create_study( + study_name="multi-dynamic", + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + x = trial.suggest_float("x1", 0, 5) + y = trial.suggest_float("y1", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + else: + x = trial.suggest_float("x2", 0, 5) + y = trial.suggest_float("y2", 0, 3) + v0 = 2 * x**2 + 2 * y**2 + v1 = (x - 2) ** 2 + (y - 3) ** 2 + return v0, v1 + + study.optimize(objective_multi_dynamic, n_trials=50) + + # Pruning with no intermediate values + study = optuna.create_study( + study_name="single-pruned-without-report", storage=storage, sampler=sampler + ) + + def objective_prune_without_report(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -15, 30) + y = trial.suggest_float("y", -15, 30) + v = x**2 + y**2 + if v > 100: + raise optuna.TrialPruned() + return v + + study.optimize(objective_prune_without_report, n_trials=100) + + # Single objective pruned after reported 'inf', '-inf', or 'nan' + study = optuna.create_study(study_name="single-inf-report", storage=storage, sampler=sampler) + + def objective_single_inf_report(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + trial.report(float("inf"), 1) + elif trial.number % 3 == 1: + trial.report(float("-inf"), 1) + else: + trial.report(float("nan"), 1) + + if x > 0: + raise optuna.TrialPruned() + else: + return x**2 + + study.optimize(objective_single_inf_report, n_trials=50) + + # Issue 410 + study = optuna.create_study(study_name="Issue 410", storage=storage, sampler=sampler) + + def objective_issue_410(trial: optuna.Trial) -> float: + trial.suggest_categorical("resample_rate", ["50ms"]) + trial.suggest_categorical("channels", ["all"]) + trial.suggest_categorical("window_size", [256]) + if trial.number > 15: + raise Exception("Unexpected error") + trial.suggest_categorical("cbow", [True]) + trial.suggest_categorical("model", ["m1"]) + + trial.set_user_attr("epochs", 0) + trial.set_user_attr("deterministic", True) + if trial.number > 10: + raise Exception("unexpeccted error") + trial.set_user_attr("folder", "/path/to/folder") + trial.set_user_attr("resample_type", "foo") + trial.set_user_attr("run_id", "0001") + return 1.0 + + study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) + + # No trials single-objective study + optuna.create_study(study_name="single-no-trials", storage=storage, sampler=sampler) + + # No trials multi-objective study + optuna.create_study( + study_name="multi-no-trials", + storage=storage, + directions=["minimize", "maximize"], + sampler=sampler, + ) + return storage + + +host = "127.0.0.1" +port = 8081 + + +@pytest.fixture(scope='session') +def server(request, dummy_storage): + app = wsgi(dummy_storage) + httpd = make_server(host, port, app) + thread = threading.Thread(target=httpd.serve_forever) + thread.start() + + def stop_server(): + httpd.shutdown() + httpd.server_close() + thread.join() + + request.addfinalizer(stop_server) + + +@pytest.fixture(scope='module') +def browser(): + with sync_playwright() as playwright: + browser = playwright.chromium.launch() + yield browser + browser.close() diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index ac7f10df..8d8c108b 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,25 +1,25 @@ -import re +import pytest from playwright.sync_api import expect from playwright.sync_api import Page +import optuna -def test_homepage_has_Playwright_in_title_and_get_started_link_linking_to_the_intro_page( - page: Page, + +def test_first( + dummy_storage: optuna.storages.InMemoryStorage, + server, + browser, ) -> None: - page.goto("https://playwright.dev/") + print(dummy_storage) + print(id(dummy_storage)) - # Expect a title "to contain" a substring. - expect(page).to_have_title(re.compile("Playwright")) +def test_second( + dummy_storage: optuna.storages.InMemoryStorage, + server, + browser, +) -> None: + page = browser.new_page() + page.goto("http://localhost:8081") + print(page.title()) - # create a locator - get_started = page.get_by_role("link", name="Get started") - - # Expect an attribute "to be strictly equal" to the value. - expect(get_started).to_have_attribute("href", "/docs/intro") - - # Click the get started link. - get_started.click() - - # Expects the URL to contain intro. - expect(page).to_have_url(re.compile(".*intro")) From 2d3959c479c2b25b097c8a9d3ea67cbb0ccfcf3a Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 11 Jun 2023 22:36:03 +0900 Subject: [PATCH 02/12] Migrate visual_regression_test --- e2e_tests/conftest.py | 30 +++++++++++--------- e2e_tests/visual_regression_test.py | 44 ++++++++++++++++++----------- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 58338334..07124540 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -1,11 +1,17 @@ -import pytest -from playwright.sync_api import sync_playwright - import threading +from typing import Generator from wsgiref.simple_server import make_server import optuna from optuna_dashboard import wsgi +from playwright.sync_api import Browser +from playwright.sync_api import sync_playwright +import pytest + + +@pytest.fixture(scope="session") +def port() -> int: + return 8081 @pytest.fixture(scope="session") @@ -180,18 +186,16 @@ def dummy_storage() -> optuna.storages.InMemoryStorage: return storage -host = "127.0.0.1" -port = 8081 - - -@pytest.fixture(scope='session') -def server(request, dummy_storage): +@pytest.fixture(scope="session") +def server( + request: pytest.FixtureRequest, dummy_storage: optuna.storages.InMemoryStorage, port: int +) -> None: app = wsgi(dummy_storage) - httpd = make_server(host, port, app) + httpd = make_server("127.0.0.1", port, app) thread = threading.Thread(target=httpd.serve_forever) thread.start() - def stop_server(): + def stop_server() -> None: httpd.shutdown() httpd.server_close() thread.join() @@ -199,8 +203,8 @@ def server(request, dummy_storage): request.addfinalizer(stop_server) -@pytest.fixture(scope='module') -def browser(): +@pytest.fixture(scope="module") +def browser() -> Generator[Browser, None, None]: with sync_playwright() as playwright: browser = playwright.chromium.launch() yield browser diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 8d8c108b..17135201 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,25 +1,35 @@ -import pytest - -from playwright.sync_api import expect -from playwright.sync_api import Page +import os import optuna +from playwright.sync_api import Browser -def test_first( - dummy_storage: optuna.storages.InMemoryStorage, - server, - browser, +output_dir = "tmp" +width = 1000 +height = 3000 + + +def test_take_screenshots_study_list( + server: None, + browser: Browser, + port: int, ) -> None: - print(dummy_storage) - print(id(dummy_storage)) + url = f"http://localhost:{port}/dashboard/" + page = browser.new_page(viewport={"width": width, "height": height}) + page.goto(url) + page.screenshot(path=os.path.join(output_dir, "study-list.png")) -def test_second( + +def test_take_screenshots_studies( dummy_storage: optuna.storages.InMemoryStorage, - server, - browser, + server: None, + browser: Browser, + port: int, ) -> None: - page = browser.new_page() - page.goto("http://localhost:8081") - print(page.title()) - + summaries = optuna.get_all_study_summaries(dummy_storage) + study_ids = {s._study_id: s.study_name for s in summaries} + for study_id, study_name in study_ids.items(): + url = f"http://localhost:{port}/dashboard/studies/{study_id}" + page = browser.new_page(viewport={"width": width, "height": height}) + page.goto(url) + page.screenshot(path=os.path.join(output_dir, f"study-{study_name}.png")) From f9b442567670a8eda9f57807b0581ebdb5286a3b Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Tue, 13 Jun 2023 23:21:46 +0900 Subject: [PATCH 03/12] Assert page title --- e2e_tests/visual_regression_test.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 17135201..61e4d2db 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,4 +1,5 @@ import os +import pytest import optuna from playwright.sync_api import Browser @@ -9,27 +10,19 @@ width = 1000 height = 3000 -def test_take_screenshots_study_list( +def test_study_list( + dummy_storage: optuna.storages.InMemoryStorage, server: None, browser: Browser, port: int, ) -> None: url = f"http://localhost:{port}/dashboard/" page = browser.new_page(viewport={"width": width, "height": height}) - page.goto(url) - page.screenshot(path=os.path.join(output_dir, "study-list.png")) - -def test_take_screenshots_studies( - dummy_storage: optuna.storages.InMemoryStorage, - server: None, - browser: Browser, - port: int, -) -> None: summaries = optuna.get_all_study_summaries(dummy_storage) study_ids = {s._study_id: s.study_name for s in summaries} for study_id, study_name in study_ids.items(): - url = f"http://localhost:{port}/dashboard/studies/{study_id}" - page = browser.new_page(viewport={"width": width, "height": height}) page.goto(url) - page.screenshot(path=os.path.join(output_dir, f"study-{study_name}.png")) + page.click(f"a[href='/dashboard/studies/{study_id}']") + title = page.query_selector('.MuiTypography-body1').text_content() + assert study_name in title \ No newline at end of file From d1872b83efebed365e9e389297a65ff261569069 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Tue, 13 Jun 2023 23:27:38 +0900 Subject: [PATCH 04/12] Fix mypy --- e2e_tests/visual_regression_test.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 61e4d2db..484603f1 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,6 +1,3 @@ -import os -import pytest - import optuna from playwright.sync_api import Browser @@ -24,5 +21,10 @@ def test_study_list( for study_id, study_name in study_ids.items(): page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}']") - title = page.query_selector('.MuiTypography-body1').text_content() - assert study_name in title \ No newline at end of file + + element = page.query_selector(".MuiTypography-body1") + assert element is not None + + title = element.text_content() + assert title is not None + assert study_name in title From a41dda1024865adfaa23ba4e2086ea38b2e274dc Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Tue, 13 Jun 2023 23:30:32 +0900 Subject: [PATCH 05/12] Fix black --- e2e_tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 07124540..c8091f0c 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -16,7 +16,6 @@ def port() -> int: @pytest.fixture(scope="session") def dummy_storage() -> optuna.storages.InMemoryStorage: - storage = optuna.storages.InMemoryStorage() sampler = optuna.samplers.RandomSampler(seed=0) From ce4809a9a647deb3fc821f5f81a1088bd23df04d Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 18 Jun 2023 13:13:23 +0900 Subject: [PATCH 06/12] Remove useless code Fix black Fix flake8 Fix comment out --- e2e_tests/conftest.py | 11 ----------- e2e_tests/visual_regression_test.py | 7 +------ 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index c8091f0c..5b8d0dc7 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -1,11 +1,8 @@ import threading -from typing import Generator from wsgiref.simple_server import make_server import optuna from optuna_dashboard import wsgi -from playwright.sync_api import Browser -from playwright.sync_api import sync_playwright import pytest @@ -200,11 +197,3 @@ def server( thread.join() request.addfinalizer(stop_server) - - -@pytest.fixture(scope="module") -def browser() -> Generator[Browser, None, None]: - with sync_playwright() as playwright: - browser = playwright.chromium.launch() - yield browser - browser.close() diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 484603f1..0405213c 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -2,11 +2,6 @@ import optuna from playwright.sync_api import Browser -output_dir = "tmp" -width = 1000 -height = 3000 - - def test_study_list( dummy_storage: optuna.storages.InMemoryStorage, server: None, @@ -14,7 +9,7 @@ def test_study_list( port: int, ) -> None: url = f"http://localhost:{port}/dashboard/" - page = browser.new_page(viewport={"width": width, "height": height}) + page = browser.new_page(viewport={"width": 1000, "height": 3000}) summaries = optuna.get_all_study_summaries(dummy_storage) study_ids = {s._study_id: s.study_name for s in summaries} From 686d4b9121164ac179f3b5afdbe104df12fff43a Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 18 Jun 2023 13:51:58 +0900 Subject: [PATCH 07/12] Use parameterize test to generate screenshot of each page --- e2e_tests/visual_regression_test.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 0405213c..e07aa952 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,25 +1,29 @@ import optuna -from playwright.sync_api import Browser +from playwright.sync_api import Page +import pytest +@pytest.mark.parametrize("study_id", range(10)) def test_study_list( + study_id: int, + page: Page, dummy_storage: optuna.storages.InMemoryStorage, - server: None, - browser: Browser, port: int, + server: None, ) -> None: + page.set_viewport_size({"width": 1000, "height": 3000}) url = f"http://localhost:{port}/dashboard/" - page = browser.new_page(viewport={"width": 1000, "height": 3000}) summaries = optuna.get_all_study_summaries(dummy_storage) study_ids = {s._study_id: s.study_name for s in summaries} - for study_id, study_name in study_ids.items(): - page.goto(url) - page.click(f"a[href='/dashboard/studies/{study_id}']") - element = page.query_selector(".MuiTypography-body1") - assert element is not None + study_name = study_ids[study_id] + page.goto(url) + page.click(f"a[href='/dashboard/studies/{study_id}']") - title = element.text_content() - assert title is not None - assert study_name in title + element = page.query_selector(".MuiTypography-body1") + assert element is not None + + title = element.text_content() + assert title is not None + assert study_name in title From 0bb14aac8d93821dc5dc4fcfb739b3780c07706f Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Tue, 20 Jun 2023 22:59:14 +0900 Subject: [PATCH 08/12] Follow review comments --- e2e_tests/conftest.py | 17 ++++++++--------- e2e_tests/visual_regression_test.py | 6 ++---- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 5b8d0dc7..e2198219 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -6,11 +6,6 @@ from optuna_dashboard import wsgi import pytest -@pytest.fixture(scope="session") -def port() -> int: - return 8081 - - @pytest.fixture(scope="session") def dummy_storage() -> optuna.storages.InMemoryStorage: storage = optuna.storages.InMemoryStorage() @@ -183,11 +178,13 @@ def dummy_storage() -> optuna.storages.InMemoryStorage: @pytest.fixture(scope="session") -def server( - request: pytest.FixtureRequest, dummy_storage: optuna.storages.InMemoryStorage, port: int -) -> None: +def server_url( + request: pytest.FixtureRequest, dummy_storage: optuna.storages.InMemoryStorage +) -> str: + ip = "127.0.0.1" + port = 38080 app = wsgi(dummy_storage) - httpd = make_server("127.0.0.1", port, app) + httpd = make_server(ip, port, app) thread = threading.Thread(target=httpd.serve_forever) thread.start() @@ -197,3 +194,5 @@ def server( thread.join() request.addfinalizer(stop_server) + + return f"http://{ip}:{port}/dashboard/" diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index e07aa952..b4091bc4 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -8,17 +8,15 @@ def test_study_list( study_id: int, page: Page, dummy_storage: optuna.storages.InMemoryStorage, - port: int, - server: None, + server_url: str, ) -> None: page.set_viewport_size({"width": 1000, "height": 3000}) - url = f"http://localhost:{port}/dashboard/" summaries = optuna.get_all_study_summaries(dummy_storage) study_ids = {s._study_id: s.study_name for s in summaries} study_name = study_ids[study_id] - page.goto(url) + page.goto(server_url) page.click(f"a[href='/dashboard/studies/{study_id}']") element = page.query_selector(".MuiTypography-body1") From f6186b554d3000dcf6054748943ebffd0cb41d6f Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 25 Jun 2023 16:01:37 +0900 Subject: [PATCH 09/12] Fix ip to addr --- e2e_tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index e2198219..5ce9d91c 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -181,10 +181,10 @@ def dummy_storage() -> optuna.storages.InMemoryStorage: def server_url( request: pytest.FixtureRequest, dummy_storage: optuna.storages.InMemoryStorage ) -> str: - ip = "127.0.0.1" + addr = "127.0.0.1" port = 38080 app = wsgi(dummy_storage) - httpd = make_server(ip, port, app) + httpd = make_server(addr, port, app) thread = threading.Thread(target=httpd.serve_forever) thread.start() @@ -195,4 +195,4 @@ def server_url( request.addfinalizer(stop_server) - return f"http://{ip}:{port}/dashboard/" + return f"http://{addr}:{port}/dashboard/" From 9bf1fb19326266540e9493a3f993bba1d61dbef0 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 25 Jun 2023 16:41:56 +0900 Subject: [PATCH 10/12] Parameterize dummy storages --- e2e_tests/conftest.py | 293 ++++++++++++++++------------ e2e_tests/visual_regression_test.py | 11 +- 2 files changed, 170 insertions(+), 134 deletions(-) diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 5ce9d91c..b9b8e36f 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -6,184 +6,223 @@ from optuna_dashboard import wsgi import pytest -@pytest.fixture(scope="session") -def dummy_storage() -> optuna.storages.InMemoryStorage: +study_names = [ + "single", + "single-trial", + "single-1-param", + "single-dynamic", + "single-inf", + "multi-objective", + "multi-dynamic", + "single-pruned-without-report", + "single-inf-report", + "issue-410", + "single-no-trials", + "multi-no-trials", +] + + +def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage: storage = optuna.storages.InMemoryStorage() sampler = optuna.samplers.RandomSampler(seed=0) - # Single-objective study - study = optuna.create_study(study_name="single", storage=storage, sampler=sampler) + # Sinble objective study + if study_name == "single": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_single(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 + def objective_single(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 - study.optimize(objective_single, n_trials=50) + study.optimize(objective_single, n_trials=50) # A single objective study with a single trial # Refs: https://github.com/optuna/optuna-dashboard/issues/401 - study = optuna.create_study(study_name="single-trial", storage=storage, sampler=sampler) - study.optimize(objective_single, n_trials=1) + elif study_name == "single-trial": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) + + def objective_single(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective_single, n_trials=1) # Single-objective study with 1 parameter - study = optuna.create_study( - study_name="single-1-param", storage=storage, direction="maximize", sampler=sampler - ) + elif study_name == "single-1-param": + study = optuna.create_study( + study_name=study_name, storage=storage, direction="maximize", sampler=sampler + ) - def objective_single_with_1param(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - return -((x1 - 2) ** 2) + def objective_single_with_1param(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + return -((x1 - 2) ** 2) - study.optimize(objective_single_with_1param, n_trials=50) + study.optimize(objective_single_with_1param, n_trials=50) # Single-objective study with dynamic search space - study = optuna.create_study( - study_name="single-dynamic", storage=storage, direction="maximize", sampler=sampler - ) + elif study_name == "single-dynamic": + study = optuna.create_study( + study_name=study_name, storage=storage, direction="maximize", sampler=sampler + ) - def objective_single_dynamic(trial: optuna.Trial) -> float: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - return (trial.suggest_float("x1", 0, 10) - 2) ** 2 - else: - return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) + def objective_single_dynamic(trial: optuna.Trial) -> float: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + return (trial.suggest_float("x1", 0, 10) - 2) ** 2 + else: + return -((trial.suggest_float("x2", -10, 0) + 5) ** 2) - study.optimize(objective_single_dynamic, n_trials=50) + study.optimize(objective_single_dynamic, n_trials=50) # Single objective study with 'inf', '-inf', or 'nan' value - study = optuna.create_study(study_name="single-inf", storage=storage, sampler=sampler) + elif study_name == "single-inf": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_single_inf(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - return float("inf") - elif trial.number % 3 == 1: - return float("-inf") - else: - return x**2 + def objective_single_inf(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + return float("inf") + elif trial.number % 3 == 1: + return float("-inf") + else: + return x**2 - study.optimize(objective_single_inf, n_trials=50) + study.optimize(objective_single_inf, n_trials=50) # Multi-objective study - study = optuna.create_study( - study_name="multi-objective", - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) + elif study_name == "multi-objective": + study = optuna.create_study( + study_name=study_name, + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) - def objective_multi(trial: optuna.Trial) -> tuple[float, float]: - x = trial.suggest_float("x", 0, 5) - y = trial.suggest_float("y", 0, 3) - v0 = 4 * x**2 + 4 * y**2 - v1 = (x - 5) ** 2 + (y - 5) ** 2 - return v0, v1 - - study.optimize(objective_multi, n_trials=50) - - # Multi-objective study with dynamic search space - study = optuna.create_study( - study_name="multi-dynamic", - storage=storage, - directions=["minimize", "minimize"], - sampler=sampler, - ) - - def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]: - category = trial.suggest_categorical("category", ["foo", "bar"]) - if category == "foo": - x = trial.suggest_float("x1", 0, 5) - y = trial.suggest_float("y1", 0, 3) + def objective_multi(trial: optuna.Trial) -> tuple[float, float]: + x = trial.suggest_float("x", 0, 5) + y = trial.suggest_float("y", 0, 3) v0 = 4 * x**2 + 4 * y**2 v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 - else: - x = trial.suggest_float("x2", 0, 5) - y = trial.suggest_float("y2", 0, 3) - v0 = 2 * x**2 + 2 * y**2 - v1 = (x - 2) ** 2 + (y - 3) ** 2 - return v0, v1 - study.optimize(objective_multi_dynamic, n_trials=50) + study.optimize(objective_multi, n_trials=50) + + # Multi-objective study with dynamic search space + elif study_name == "multi-dynamic": + study = optuna.create_study( + study_name=study_name, + storage=storage, + directions=["minimize", "minimize"], + sampler=sampler, + ) + + def objective_multi_dynamic(trial: optuna.Trial) -> tuple[float, float]: + category = trial.suggest_categorical("category", ["foo", "bar"]) + if category == "foo": + x = trial.suggest_float("x1", 0, 5) + y = trial.suggest_float("y1", 0, 3) + v0 = 4 * x**2 + 4 * y**2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + else: + x = trial.suggest_float("x2", 0, 5) + y = trial.suggest_float("y2", 0, 3) + v0 = 2 * x**2 + 2 * y**2 + v1 = (x - 2) ** 2 + (y - 3) ** 2 + return v0, v1 + + study.optimize(objective_multi_dynamic, n_trials=50) # Pruning with no intermediate values - study = optuna.create_study( - study_name="single-pruned-without-report", storage=storage, sampler=sampler - ) + elif study_name == "single-pruned-without-report": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_prune_without_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -15, 30) - y = trial.suggest_float("y", -15, 30) - v = x**2 + y**2 - if v > 100: - raise optuna.TrialPruned() - return v + def objective_prune_without_report(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -15, 30) + y = trial.suggest_float("y", -15, 30) + v = x**2 + y**2 + if v > 100: + raise optuna.TrialPruned() + return v - study.optimize(objective_prune_without_report, n_trials=100) + study.optimize(objective_prune_without_report, n_trials=100) # Single objective pruned after reported 'inf', '-inf', or 'nan' - study = optuna.create_study(study_name="single-inf-report", storage=storage, sampler=sampler) + elif study_name == "single-inf-report": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_single_inf_report(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - trial.report(float("inf"), 1) - elif trial.number % 3 == 1: - trial.report(float("-inf"), 1) - else: - trial.report(float("nan"), 1) + def objective_single_inf_report(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -10, 10) + if trial.number % 3 == 0: + trial.report(float("inf"), 1) + elif trial.number % 3 == 1: + trial.report(float("-inf"), 1) + else: + trial.report(float("nan"), 1) - if x > 0: - raise optuna.TrialPruned() - else: - return x**2 + if x > 0: + raise optuna.TrialPruned() + else: + return x**2 - study.optimize(objective_single_inf_report, n_trials=50) + study.optimize(objective_single_inf_report, n_trials=50) # Issue 410 - study = optuna.create_study(study_name="Issue 410", storage=storage, sampler=sampler) + elif study_name == "issue-410": + study = optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) - def objective_issue_410(trial: optuna.Trial) -> float: - trial.suggest_categorical("resample_rate", ["50ms"]) - trial.suggest_categorical("channels", ["all"]) - trial.suggest_categorical("window_size", [256]) - if trial.number > 15: - raise Exception("Unexpected error") - trial.suggest_categorical("cbow", [True]) - trial.suggest_categorical("model", ["m1"]) + def objective_issue_410(trial: optuna.Trial) -> float: + trial.suggest_categorical("resample_rate", ["50ms"]) + trial.suggest_categorical("channels", ["all"]) + trial.suggest_categorical("window_size", [256]) + if trial.number > 15: + raise Exception("Unexpected error") + trial.suggest_categorical("cbow", [True]) + trial.suggest_categorical("model", ["m1"]) - trial.set_user_attr("epochs", 0) - trial.set_user_attr("deterministic", True) - if trial.number > 10: - raise Exception("unexpeccted error") - trial.set_user_attr("folder", "/path/to/folder") - trial.set_user_attr("resample_type", "foo") - trial.set_user_attr("run_id", "0001") - return 1.0 + trial.set_user_attr("epochs", 0) + trial.set_user_attr("deterministic", True) + if trial.number > 10: + raise Exception("unexpeccted error") + trial.set_user_attr("folder", "/path/to/folder") + trial.set_user_attr("resample_type", "foo") + trial.set_user_attr("run_id", "0001") + return 1.0 - study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) + study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) # No trials single-objective study - optuna.create_study(study_name="single-no-trials", storage=storage, sampler=sampler) + elif study_name == "single-no-trials": + optuna.create_study(study_name=study_name, storage=storage, sampler=sampler) # No trials multi-objective study - optuna.create_study( - study_name="multi-no-trials", - storage=storage, - directions=["minimize", "maximize"], - sampler=sampler, - ) + elif study_name == "multi-no-trials": + optuna.create_study( + study_name=study_name, + storage=storage, + directions=["minimize", "maximize"], + sampler=sampler, + ) + else: + assert False, f"No study configuration of {study_name} in conftest.py" + + return storage + + +@pytest.fixture(scope="session", params=study_names) +def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage: + study_name = request.param + storage = make_dummy_storage(study_name) return storage @pytest.fixture(scope="session") -def server_url( - request: pytest.FixtureRequest, dummy_storage: optuna.storages.InMemoryStorage -) -> str: +def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: addr = "127.0.0.1" port = 38080 - app = wsgi(dummy_storage) + app = wsgi(storage) httpd = make_server(addr, port, app) thread = threading.Thread(target=httpd.serve_forever) thread.start() diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index b4091bc4..42b2190c 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -1,21 +1,18 @@ import optuna from playwright.sync_api import Page -import pytest -@pytest.mark.parametrize("study_id", range(10)) def test_study_list( - study_id: int, page: Page, - dummy_storage: optuna.storages.InMemoryStorage, + storage: optuna.storages.InMemoryStorage, server_url: str, ) -> None: page.set_viewport_size({"width": 1000, "height": 3000}) - summaries = optuna.get_all_study_summaries(dummy_storage) - study_ids = {s._study_id: s.study_name for s in summaries} + summaries = optuna.get_all_study_summaries(storage) + study_id = summaries[0]._study_id + study_name = summaries[0].study_name - study_name = study_ids[study_id] page.goto(server_url) page.click(f"a[href='/dashboard/studies/{study_id}']") From f9050de403e2f76bb052bf4f342c6b82f8aea4d1 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 25 Jun 2023 18:01:39 +0900 Subject: [PATCH 11/12] Add e2e tests for the other pages --- e2e_tests/conftest.py | 24 +++++++- e2e_tests/visual_regression_test.py | 86 ++++++++++++++++++++++++++++- optuna_dashboard/ts/searchSpace.ts | 13 +++-- 3 files changed, 114 insertions(+), 9 deletions(-) diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index b9b8e36f..8d8eac3b 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -1,3 +1,4 @@ +import socket import threading from wsgiref.simple_server import make_server @@ -211,6 +212,14 @@ def make_dummy_storage(study_name: str) -> optuna.storages.InMemoryStorage: return storage +def get_free_port() -> int: + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.bind(("", 0)) + _, port = tcp.getsockname() + tcp.close() + return port + + @pytest.fixture(scope="session", params=study_names) def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage: study_name = request.param @@ -221,7 +230,7 @@ def storage(request: pytest.FixtureRequest) -> optuna.storages.InMemoryStorage: @pytest.fixture(scope="session") def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemoryStorage) -> str: addr = "127.0.0.1" - port = 38080 + port = get_free_port() app = wsgi(storage) httpd = make_server(addr, port, app) thread = threading.Thread(target=httpd.serve_forever) @@ -234,4 +243,15 @@ def server_url(request: pytest.FixtureRequest, storage: optuna.storages.InMemory request.addfinalizer(stop_server) - return f"http://{addr}:{port}/dashboard/" + return f"http://{addr}:{port}/dashboard" + + +@pytest.fixture(scope="session") +def browser_context_args(browser_context_args: dict) -> dict: + return { + **browser_context_args, + "viewport": { + "width": 1000, + "height": 3000, + }, + } diff --git a/e2e_tests/visual_regression_test.py b/e2e_tests/visual_regression_test.py index 42b2190c..706ec2f3 100644 --- a/e2e_tests/visual_regression_test.py +++ b/e2e_tests/visual_regression_test.py @@ -7,8 +7,6 @@ def test_study_list( storage: optuna.storages.InMemoryStorage, server_url: str, ) -> None: - page.set_viewport_size({"width": 1000, "height": 3000}) - summaries = optuna.get_all_study_summaries(storage) study_id = summaries[0]._study_id study_name = summaries[0].study_name @@ -22,3 +20,87 @@ def test_study_list( title = element.text_content() assert title is not None assert study_name in title + + +def test_study_analytics( + page: Page, + storage: optuna.storages.InMemoryStorage, + server_url: str, +) -> None: + summaries = optuna.get_all_study_summaries(storage) + study_id = summaries[0]._study_id + study_name = summaries[0].study_name + url = f"{server_url}/studies/{study_id}" + + page.goto(url) + page.click(f"a[href='/dashboard/studies/{study_id}/analytics']") + + element = page.query_selector(".MuiTypography-body1") + assert element is not None + + title = element.text_content() + assert title is not None + assert study_name in title + + +def test_trial_list( + page: Page, + storage: optuna.storages.InMemoryStorage, + server_url: str, +) -> None: + summaries = optuna.get_all_study_summaries(storage) + study_id = summaries[0]._study_id + study_name = summaries[0].study_name + url = f"{server_url}/studies/{study_id}" + + page.goto(url) + page.click(f"a[href='/dashboard/studies/{study_id}/trials']") + + element = page.query_selector(".MuiTypography-body1") + assert element is not None + + title = element.text_content() + assert title is not None + assert study_name in title + + +def test_trial_table( + page: Page, + storage: optuna.storages.InMemoryStorage, + server_url: str, +) -> None: + summaries = optuna.get_all_study_summaries(storage) + study_id = summaries[0]._study_id + study_name = summaries[0].study_name + url = f"{server_url}/studies/{study_id}" + + page.goto(url) + page.click(f"a[href='/dashboard/studies/{study_id}/trialTable']") + + element = page.query_selector(".MuiTypography-body1") + assert element is not None + + title = element.text_content() + assert title is not None + assert study_name in title + + +def test_trial_note( + page: Page, + storage: optuna.storages.InMemoryStorage, + server_url: str, +) -> None: + summaries = optuna.get_all_study_summaries(storage) + study_id = summaries[0]._study_id + study_name = summaries[0].study_name + url = f"{server_url}/studies/{study_id}" + + page.goto(url) + page.click(f"a[href='/dashboard/studies/{study_id}/note']") + + element = page.query_selector(".MuiTypography-body1") + assert element is not None + + title = element.text_content() + assert title is not None + assert study_name in title diff --git a/optuna_dashboard/ts/searchSpace.ts b/optuna_dashboard/ts/searchSpace.ts index 729f1f51..f0bd4006 100644 --- a/optuna_dashboard/ts/searchSpace.ts +++ b/optuna_dashboard/ts/searchSpace.ts @@ -3,9 +3,9 @@ import { useMemo } from "react" export const mergeUnionSearchSpace = ( unionSearchSpace: SearchSpaceItem[] ): SearchSpaceItem[] => { - const knownElements = new Map() + let knownElements = new Map() unionSearchSpace.forEach((s) => { - const d = knownElements.get(s.name) + let d = knownElements.get(s.name) if (d === undefined) { knownElements.set(s.name, s.distribution) return @@ -17,9 +17,12 @@ export const mergeUnionSearchSpace = ( // CategoricalDistribution.choices will never be changed return } - d.low = Math.min(d.low, s.distribution.low) - d.high = Math.max(d.low, s.distribution.high) - knownElements.set(s.name, d) + const updated: Distribution = { + ...d, + low: Math.min(d.low, s.distribution.low), + high: Math.max(d.high, s.distribution.high), + } + knownElements.set(s.name, updated) }) return Array.from(knownElements.keys()) .sort((a, b) => (a > b ? 1 : a < b ? -1 : 0)) From e4244fb141aee2247f903eebafef978db5a382f2 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 25 Jun 2023 18:03:39 +0900 Subject: [PATCH 12/12] Revert changes --- optuna_dashboard/ts/searchSpace.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/ts/searchSpace.ts b/optuna_dashboard/ts/searchSpace.ts index f0bd4006..daf79b11 100644 --- a/optuna_dashboard/ts/searchSpace.ts +++ b/optuna_dashboard/ts/searchSpace.ts @@ -3,9 +3,9 @@ import { useMemo } from "react" export const mergeUnionSearchSpace = ( unionSearchSpace: SearchSpaceItem[] ): SearchSpaceItem[] => { - let knownElements = new Map() + const knownElements = new Map() unionSearchSpace.forEach((s) => { - let d = knownElements.get(s.name) + const d = knownElements.get(s.name) if (d === undefined) { knownElements.set(s.name, s.distribution) return