From 5f91aa4672833464ed149fab934d2b2e83feec00 Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 8 Apr 2021 01:44:20 +0900 Subject: [PATCH 1/4] Run integration test by using pypputeer --- .gitignore | 1 + integration_tests.py | 66 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 integration_tests.py diff --git a/.gitignore b/.gitignore index 11b238fc..05f2bf7f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ public/ *.sqlite3 *.db +*.png diff --git a/integration_tests.py b/integration_tests.py new file mode 100644 index 00000000..26fec354 --- /dev/null +++ b/integration_tests.py @@ -0,0 +1,66 @@ +import asyncio +import optuna +import time +import threading + +from optuna_dashboard.app import create_app +from pyppeteer import launch +from pyppeteer.browser import Browser +from wsgiref.simple_server import make_server + + +host = "127.0.0.1" +port = 8080 + + +async def take_screenshot(browser: Browser, url: str, path: str): + page = await browser.newPage() + await page.setViewport({'width': 1200, 'height': 3000}) + await page.goto(url) + time.sleep(5) + await page.screenshot({'path': path}) + + +async def integration_test_main(): + browser = await launch() + await take_screenshot(browser, f'http://{host}:{port}/dashboard/', 'study_list.png') + await take_screenshot(browser, f'http://{host}:{port}/dashboard/studies/0', 'study_detail_0.png') + await take_screenshot(browser, f'http://{host}:{port}/dashboard/studies/1', 'study_detail_1.png') + await browser.close() + + +def run_optuna() -> optuna.storages.InMemoryStorage: + storage = optuna.storages.InMemoryStorage() + + # study 0 + study0 = optuna.create_study(study_name="example-0", storage=storage) + + def objective(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study0.optimize(objective, n_trials=100) + + # study 1 - no trials + optuna.create_study(study_name="example-1", storage=storage) + return storage + + +def main(): + storage = run_optuna() + app = create_app(storage) + + httpd = make_server(host, port, app) + thread = threading.Thread(target=httpd.serve_forever) + thread.start() + + asyncio.get_event_loop().run_until_complete(integration_test_main()) + + httpd.shutdown() + httpd.server_close() + thread.join() + + +if __name__ == '__main__': + main() From a823d8bd40eff185e6cb388f9e4bd7d5ef1cded3 Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 8 Apr 2021 01:50:24 +0900 Subject: [PATCH 2/4] Fix lint errors --- integration_tests.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/integration_tests.py b/integration_tests.py index 26fec354..a1941f95 100644 --- a/integration_tests.py +++ b/integration_tests.py @@ -8,31 +8,38 @@ from pyppeteer import launch from pyppeteer.browser import Browser from wsgiref.simple_server import make_server - host = "127.0.0.1" port = 8080 -async def take_screenshot(browser: Browser, url: str, path: str): +async def take_screenshot( + browser: Browser, url: str, path: str, sleep_sec: int = 5 +) -> None: page = await browser.newPage() - await page.setViewport({'width': 1200, 'height': 3000}) + await page.setViewport({"width": 1200, "height": 3000}) await page.goto(url) - time.sleep(5) - await page.screenshot({'path': path}) + time.sleep(sleep_sec) + await page.screenshot({"path": path}) -async def integration_test_main(): +async def integration_test_main() -> None: browser = await launch() - await take_screenshot(browser, f'http://{host}:{port}/dashboard/', 'study_list.png') - await take_screenshot(browser, f'http://{host}:{port}/dashboard/studies/0', 'study_detail_0.png') - await take_screenshot(browser, f'http://{host}:{port}/dashboard/studies/1', 'study_detail_1.png') + await take_screenshot( + browser, f"http://{host}:{port}/dashboard/", "study_list.png", sleep_sec=1 + ) + await take_screenshot( + browser, f"http://{host}:{port}/dashboard/studies/0", "study_detail_0.png" + ) + await take_screenshot( + browser, f"http://{host}:{port}/dashboard/studies/1", "study_detail_1.png" + ) await browser.close() def run_optuna() -> optuna.storages.InMemoryStorage: storage = optuna.storages.InMemoryStorage() - # study 0 + # study (study_id=0) study0 = optuna.create_study(study_name="example-0", storage=storage) def objective(trial: optuna.Trial) -> float: @@ -42,12 +49,12 @@ def run_optuna() -> optuna.storages.InMemoryStorage: study0.optimize(objective, n_trials=100) - # study 1 - no trials + # study (study_id=1) with no trials optuna.create_study(study_name="example-1", storage=storage) return storage -def main(): +def main() -> None: storage = run_optuna() app = create_app(storage) @@ -62,5 +69,5 @@ def main(): thread.join() -if __name__ == '__main__': +if __name__ == "__main__": main() From b6773e72170defb04667d8f506dad153afa8115d Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 8 Apr 2021 17:00:33 +0900 Subject: [PATCH 3/4] Update visual regression test --- integration_tests.py | 73 ------------------------------ visual_regression_test.py | 94 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 73 deletions(-) delete mode 100644 integration_tests.py create mode 100644 visual_regression_test.py diff --git a/integration_tests.py b/integration_tests.py deleted file mode 100644 index a1941f95..00000000 --- a/integration_tests.py +++ /dev/null @@ -1,73 +0,0 @@ -import asyncio -import optuna -import time -import threading - -from optuna_dashboard.app import create_app -from pyppeteer import launch -from pyppeteer.browser import Browser -from wsgiref.simple_server import make_server - -host = "127.0.0.1" -port = 8080 - - -async def take_screenshot( - browser: Browser, url: str, path: str, sleep_sec: int = 5 -) -> None: - page = await browser.newPage() - await page.setViewport({"width": 1200, "height": 3000}) - await page.goto(url) - time.sleep(sleep_sec) - await page.screenshot({"path": path}) - - -async def integration_test_main() -> None: - browser = await launch() - await take_screenshot( - browser, f"http://{host}:{port}/dashboard/", "study_list.png", sleep_sec=1 - ) - await take_screenshot( - browser, f"http://{host}:{port}/dashboard/studies/0", "study_detail_0.png" - ) - await take_screenshot( - browser, f"http://{host}:{port}/dashboard/studies/1", "study_detail_1.png" - ) - await browser.close() - - -def run_optuna() -> optuna.storages.InMemoryStorage: - storage = optuna.storages.InMemoryStorage() - - # study (study_id=0) - study0 = optuna.create_study(study_name="example-0", storage=storage) - - def objective(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 - - study0.optimize(objective, n_trials=100) - - # study (study_id=1) with no trials - optuna.create_study(study_name="example-1", storage=storage) - return storage - - -def main() -> None: - storage = run_optuna() - app = create_app(storage) - - httpd = make_server(host, port, app) - thread = threading.Thread(target=httpd.serve_forever) - thread.start() - - asyncio.get_event_loop().run_until_complete(integration_test_main()) - - httpd.shutdown() - httpd.server_close() - thread.join() - - -if __name__ == "__main__": - main() diff --git a/visual_regression_test.py b/visual_regression_test.py new file mode 100644 index 00000000..1142d856 --- /dev/null +++ b/visual_regression_test.py @@ -0,0 +1,94 @@ +import asyncio +import threading +import time +import optuna +import os + +from optuna_dashboard.app import create_app +from pyppeteer import launch +from wsgiref.simple_server import make_server + +host = "127.0.0.1" +port = 8080 +output_dir = "tmp" + + +def create_optuna_storage() -> optuna.storages.InMemoryStorage: + storage = optuna.storages.InMemoryStorage() + + # Single-objective study + study = optuna.create_study(study_name="single-objective", storage=storage) + + def objective(trial: optuna.Trial) -> float: + x1 = trial.suggest_float("x1", 0, 10) + x2 = trial.suggest_float("x2", 0, 10) + return (x1 - 2) ** 2 + (x2 - 5) ** 2 + + study.optimize(objective, n_trials=100) + + # Multi-objective study + study = optuna.create_study(study_name="multi-objective", storage=storage, directions=["minimize", "minimize"]) + + def objective(trial): + x = trial.suggest_float("x", 0, 5) + y = trial.suggest_float("y", 0, 3) + v0 = 4 * x ** 2 + 4 * y ** 2 + v1 = (x - 5) ** 2 + (y - 5) ** 2 + return v0, v1 + + study.optimize(objective, n_trials=50) + + # Pruning with no intermediate values + study = optuna.create_study(study_name="binh-korn-function-with-constraints", storage=storage) + + def objective(trial): + x = trial.suggest_float("x", -15, 30) + y = trial.suggest_float("y", -15, 30) + v = x ** 2 + y ** 2 + if v > 100: + raise optuna.TrialPruned() + return v + + study.optimize(objective, n_trials=100) + + # No trials + optuna.create_study(study_name="no trials", storage=storage) + return storage + + +async def take_screenshots(study_ids) -> None: + browser = await launch() + page = await browser.newPage() + await page.setViewport({"width": 1200, "height": 3000}) + + await page.goto(f"http://{host}:{port}/dashboard/") + time.sleep(1) + await page.screenshot({"path": os.path.join(output_dir, "study-list.png")}) + + for study_id in study_ids: + await page.goto(f"http://{host}:{port}/dashboard/studies/{study_id}") + time.sleep(5) + await page.screenshot({"path": os.path.join(output_dir, f"study-{study_id}.png")}) + await browser.close() + + +def main() -> None: + os.makedirs(output_dir, exist_ok=True) + + storage = create_optuna_storage() + app = create_app(storage) + httpd = make_server(host, port, app) + thread = threading.Thread(target=httpd.serve_forever) + thread.start() + + study_ids = [s._study_id for s in storage.get_all_study_summaries()] + loop = asyncio.get_event_loop() + loop.run_until_complete(take_screenshots(study_ids)) + + httpd.shutdown() + httpd.server_close() + thread.join() + + +if __name__ == "__main__": + main() From 7348c760d8abb3bbe2e5548731830496ae0364af Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 8 Apr 2021 17:04:52 +0900 Subject: [PATCH 4/4] Fix lint errors --- .gitignore | 2 +- visual_regression_test.py | 29 +++++++++++++++++++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 05f2bf7f..cbbc6b30 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ build/ node_modules/ public/ .tox/ +tmp/ *.sqlite3 *.db -*.png diff --git a/visual_regression_test.py b/visual_regression_test.py index 1142d856..e7d56922 100644 --- a/visual_regression_test.py +++ b/visual_regression_test.py @@ -6,6 +6,7 @@ import os from optuna_dashboard.app import create_app from pyppeteer import launch +from typing import List, Tuple from wsgiref.simple_server import make_server host = "127.0.0.1" @@ -19,29 +20,35 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage: # Single-objective study study = optuna.create_study(study_name="single-objective", storage=storage) - def objective(trial: optuna.Trial) -> float: + def objective_single(trial: optuna.Trial) -> float: x1 = trial.suggest_float("x1", 0, 10) x2 = trial.suggest_float("x2", 0, 10) return (x1 - 2) ** 2 + (x2 - 5) ** 2 - study.optimize(objective, n_trials=100) + study.optimize(objective_single, n_trials=100) # Multi-objective study - study = optuna.create_study(study_name="multi-objective", storage=storage, directions=["minimize", "minimize"]) + study = optuna.create_study( + study_name="multi-objective", + storage=storage, + directions=["minimize", "minimize"], + ) - def objective(trial): + def objective_multi(trial: optuna.Trial) -> Tuple[float, float]: x = trial.suggest_float("x", 0, 5) y = trial.suggest_float("y", 0, 3) v0 = 4 * x ** 2 + 4 * y ** 2 v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 - study.optimize(objective, n_trials=50) + study.optimize(objective_multi, n_trials=50) # Pruning with no intermediate values - study = optuna.create_study(study_name="binh-korn-function-with-constraints", storage=storage) + study = optuna.create_study( + study_name="binh-korn-function-with-constraints", storage=storage + ) - def objective(trial): + def objective_prune_with_no_trials(trial: optuna.Trial) -> float: x = trial.suggest_float("x", -15, 30) y = trial.suggest_float("y", -15, 30) v = x ** 2 + y ** 2 @@ -49,14 +56,14 @@ def create_optuna_storage() -> optuna.storages.InMemoryStorage: raise optuna.TrialPruned() return v - study.optimize(objective, n_trials=100) + study.optimize(objective_prune_with_no_trials, n_trials=100) # No trials optuna.create_study(study_name="no trials", storage=storage) return storage -async def take_screenshots(study_ids) -> None: +async def take_screenshots(study_ids: List[int]) -> None: browser = await launch() page = await browser.newPage() await page.setViewport({"width": 1200, "height": 3000}) @@ -68,7 +75,9 @@ async def take_screenshots(study_ids) -> None: for study_id in study_ids: await page.goto(f"http://{host}:{port}/dashboard/studies/{study_id}") time.sleep(5) - await page.screenshot({"path": os.path.join(output_dir, f"study-{study_id}.png")}) + await page.screenshot( + {"path": os.path.join(output_dir, f"study-{study_id}.png")} + ) await browser.close()