From 3292b06484ba91cd75a8f73bb808ea65a0d9242f Mon Sep 17 00:00:00 2001 From: c-bata Date: Mon, 19 Aug 2024 16:05:53 +0900 Subject: [PATCH 1/4] Port some e2e test scenarios to vitest --- .github/workflows/e2e-dashboard-tests.yml | 10 +++++-- .github/workflows/e2e-standalone-tests.yml | 6 +++- .../test_dashboard/visual_regression_test.py | 28 ------------------- tslib/react/test/generate_assets.py | 24 ++++++++++++++++ 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/.github/workflows/e2e-dashboard-tests.yml b/.github/workflows/e2e-dashboard-tests.yml index 0f293273..71d77f0e 100644 --- a/.github/workflows/e2e-dashboard-tests.yml +++ b/.github/workflows/e2e-dashboard-tests.yml @@ -6,8 +6,12 @@ on: paths: - '.github/workflows/e2e-dashboard-tests.yml' - '**.py' - - '**.ts' - - '**.tsx' + - 'tslib/**.ts' + - 'tslib/**.tsx' + - 'tslib/**/package.json' + - 'tslib/**/package-lock.json' + - 'optuna_dashboard/**.ts' + - 'optuna_dashboard/**.tsx' - 'optuna_dashboard/package.json' - 'optuna_dashboard/package-lock.json' - 'optuna_dashboard/tsconfig.json' @@ -37,7 +41,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.11' architecture: x64 - name: Setup Optuna ${{ matrix.optuna-version }} diff --git a/.github/workflows/e2e-standalone-tests.yml b/.github/workflows/e2e-standalone-tests.yml index d449e2b7..a599336b 100644 --- a/.github/workflows/e2e-standalone-tests.yml +++ b/.github/workflows/e2e-standalone-tests.yml @@ -5,6 +5,10 @@ on: - main paths: - '.github/workflows/e2e-standalone-tests.yml' + - 'tslib/**.ts' + - 'tslib/**.tsx' + - 'tslib/**/package.json' + - 'tslib/**/package-lock.json' - 'standalone_app/**.ts' - 'standalone_app/**.tsx' - 'standalone_app/package.json' @@ -34,7 +38,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.11' architecture: x64 - name: Setup Optuna ${{ matrix.optuna-version }} diff --git a/e2e_tests/test_dashboard/visual_regression_test.py b/e2e_tests/test_dashboard/visual_regression_test.py index ea6ff930..3ecb959d 100644 --- a/e2e_tests/test_dashboard/visual_regression_test.py +++ b/e2e_tests/test_dashboard/visual_regression_test.py @@ -192,33 +192,6 @@ def run_single_inf_report_objective_study( return study -def run_issue_410_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: - # Issue 410 - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study(study_name="issue-410", storage=storage, sampler=sampler) - - def objective(trial: optuna.Trial) -> float: - trial.suggest_categorical("resample_rate", ["50ms"]) - trial.suggest_categorical("channels", ["all"]) - trial.suggest_categorical("window_size", [256]) - if trial.number > 15: - raise Exception("Unexpected error") - trial.suggest_categorical("cbow", [True]) - trial.suggest_categorical("model", ["m1"]) - - trial.set_user_attr("epochs", 0) - trial.set_user_attr("deterministic", True) - if trial.number > 10: - raise Exception("unexpeccted error") - trial.set_user_attr("folder", "/path/to/folder") - trial.set_user_attr("resample_type", "foo") - trial.set_user_attr("run_id", "0001") - return 1.0 - - study.optimize(objective, n_trials=20, catch=(Exception,)) - return study - - def run_single_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: # No trials single-objective study sampler = optuna.samplers.RandomSampler(seed=0) @@ -251,7 +224,6 @@ parameterize_studies = pytest.mark.parametrize( run_multi_dynamic_objective_study, run_single_pruned_without_report_objective_study, run_single_inf_report_objective_study, - run_issue_410_objective_study, run_single_no_trials_objective_study, run_multi_no_trials_objective_study, ], diff --git a/tslib/react/test/generate_assets.py b/tslib/react/test/generate_assets.py index f7bdc33f..49334263 100644 --- a/tslib/react/test/generate_assets.py +++ b/tslib/react/test/generate_assets.py @@ -447,6 +447,30 @@ def create_optuna_storage( trial.report(trial.number, step=0) trial.report(trial.number + 1, step=1) + # optuna-dashboard issue 410 + # https://github.com/optuna/optuna-dashboard/issues/410 + study = optuna.create_study(study_name="optuna-dashboard-issue-410", storage=storage, sampler=optuna.samplers.RandomSampler()) + + def objective_issue_410(trial: optuna.Trial) -> float: + trial.suggest_categorical("resample_rate", ["50ms"]) + trial.suggest_categorical("channels", ["all"]) + trial.suggest_categorical("window_size", [256]) + if trial.number > 15: + raise Exception("Unexpected error") + trial.suggest_categorical("cbow", [True]) + trial.suggest_categorical("model", ["m1"]) + + trial.set_user_attr("epochs", 0) + trial.set_user_attr("deterministic", True) + if trial.number > 10: + raise Exception("unexpeccted error") + trial.set_user_attr("folder", "/path/to/folder") + trial.set_user_attr("resample_type", "foo") + trial.set_user_attr("run_id", "0001") + return 1.0 + + study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) + def main() -> None: remove_assets() From c2bee72b1f78a248c4795c6b4467f1dc075461ff Mon Sep 17 00:00:00 2001 From: c-bata Date: Mon, 19 Aug 2024 18:50:59 +0900 Subject: [PATCH 2/4] Use page.wait_for_selector to fix e2e tests --- .../test_dashboard/visual_regression_test.py | 74 ++----------------- tslib/react/test/generate_assets.py | 6 +- 2 files changed, 10 insertions(+), 70 deletions(-) diff --git a/e2e_tests/test_dashboard/visual_regression_test.py b/e2e_tests/test_dashboard/visual_regression_test.py index 3ecb959d..1cd1127f 100644 --- a/e2e_tests/test_dashboard/visual_regression_test.py +++ b/e2e_tests/test_dashboard/visual_regression_test.py @@ -46,20 +46,6 @@ def run_single_trial_objective_study(storage: optuna.storages.InMemoryStorage) - return study -def run_single_1param_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study( - study_name="single-1-param", storage=storage, direction="maximize", sampler=sampler - ) - - def objective(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - return -((x1 - 2) ** 2) - - study.optimize(objective, n_trials=50) - return study - - def run_single_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: # Single-objective study with dynamic search space sampler = optuna.samplers.RandomSampler(seed=0) @@ -78,24 +64,6 @@ def run_single_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) return study -def run_single_inf_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: - # Single objective study with 'inf', '-inf', or 'nan' value - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study(study_name="single-inf", storage=storage, sampler=sampler) - - def objective(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - return float("inf") - elif trial.number % 3 == 1: - return float("-inf") - else: - return x**2 - - study.optimize(objective, n_trials=50) - return study - - def run_multi_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: # Multi-objective study sampler = optuna.samplers.RandomSampler(seed=0) @@ -167,39 +135,6 @@ def run_single_pruned_without_report_objective_study( return study -def run_single_inf_report_objective_study( - storage: optuna.storages.InMemoryStorage, -) -> optuna.Study: - # Single objective pruned after reported 'inf', '-inf', or 'nan' - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study(study_name="single-inf-report", storage=storage, sampler=sampler) - - def objective(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - trial.report(float("inf"), 1) - elif trial.number % 3 == 1: - trial.report(float("-inf"), 1) - else: - trial.report(float("nan"), 1) - - if x > 0: - raise optuna.TrialPruned() - else: - return x**2 - - study.optimize(objective, n_trials=50) - return study - - -def run_single_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: - # No trials single-objective study - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study(study_name="single-no-trials", storage=storage, sampler=sampler) - - return study - - def run_multi_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: # No trials multi-objective study sampler = optuna.samplers.RandomSampler(seed=0) @@ -217,14 +152,10 @@ parameterize_studies = pytest.mark.parametrize( [ run_single_objective_study, run_single_trial_objective_study, - run_single_1param_objective_study, run_single_dynamic_objective_study, - run_single_inf_objective_study, run_multi_objective_study, run_multi_dynamic_objective_study, run_single_pruned_without_report_objective_study, - run_single_inf_report_objective_study, - run_single_no_trials_objective_study, run_multi_no_trials_objective_study, ], ) @@ -245,6 +176,7 @@ def test_study_list( page.goto(server_url) page.click(f"a[href='/dashboard/studies/{study_id}']") + page.wait_for_selector(".MuiTypography-body1") element = page.query_selector(".MuiTypography-body1") assert element is not None @@ -269,6 +201,7 @@ def test_study_analytics( page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/analytics']") + page.wait_for_selector(".MuiTypography-body1") element = page.query_selector(".MuiTypography-body1") assert element is not None @@ -293,6 +226,7 @@ def test_trial_list( page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/trials']") + page.wait_for_selector(".MuiTypography-body1") element = page.query_selector(".MuiTypography-body1") assert element is not None @@ -317,6 +251,7 @@ def test_trial_table( page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/trialTable']") + page.wait_for_selector(".MuiTypography-body1") element = page.query_selector(".MuiTypography-body1") assert element is not None @@ -339,6 +274,7 @@ def test_trial_note( url = f"{server_url}/studies/{study_id}" page.goto(url) + page.wait_for_selector(".MuiTypography-body1") page.click(f"a[href='/dashboard/studies/{study_id}/note']") element = page.query_selector(".MuiTypography-body1") diff --git a/tslib/react/test/generate_assets.py b/tslib/react/test/generate_assets.py index 49334263..0390bfd1 100644 --- a/tslib/react/test/generate_assets.py +++ b/tslib/react/test/generate_assets.py @@ -449,7 +449,11 @@ def create_optuna_storage( # optuna-dashboard issue 410 # https://github.com/optuna/optuna-dashboard/issues/410 - study = optuna.create_study(study_name="optuna-dashboard-issue-410", storage=storage, sampler=optuna.samplers.RandomSampler()) + study = optuna.create_study( + study_name="optuna-dashboard-issue-410", + storage=storage, + sampler=optuna.samplers.RandomSampler(), + ) def objective_issue_410(trial: optuna.Trial) -> float: trial.suggest_categorical("resample_rate", ["50ms"]) From 1d346af779a81370a122d173d3fffd50d562f951 Mon Sep 17 00:00:00 2001 From: c-bata Date: Mon, 19 Aug 2024 19:11:36 +0900 Subject: [PATCH 3/4] Fix e2e tests --- e2e_tests/test_dashboard/visual_regression_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e_tests/test_dashboard/visual_regression_test.py b/e2e_tests/test_dashboard/visual_regression_test.py index 1cd1127f..7bf61356 100644 --- a/e2e_tests/test_dashboard/visual_regression_test.py +++ b/e2e_tests/test_dashboard/visual_regression_test.py @@ -27,7 +27,7 @@ def run_single_objective_study(storage: optuna.storages.InMemoryStorage) -> optu x2 = trial.suggest_float("x2", 0, 10) return (x1 - 2) ** 2 + (x2 - 5) ** 2 - study.optimize(objective, n_trials=50) + study.optimize(objective, n_trials=20) return study @@ -81,7 +81,7 @@ def run_multi_objective_study(storage: optuna.storages.InMemoryStorage) -> optun v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 - study.optimize(objective, n_trials=50) + study.optimize(objective, n_trials=20) return study @@ -110,7 +110,7 @@ def run_multi_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) v1 = (x - 2) ** 2 + (y - 3) ** 2 return v0, v1 - study.optimize(objective, n_trials=50) + study.optimize(objective, n_trials=20) return study @@ -201,7 +201,7 @@ def test_study_analytics( page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/analytics']") - page.wait_for_selector(".MuiTypography-body1") + page.wait_for_selector(".MuiTypography-body1", timeout=60 * 1000) element = page.query_selector(".MuiTypography-body1") assert element is not None From 11b0c0b8e0836b292b6a584450a450c76178fcf0 Mon Sep 17 00:00:00 2001 From: c-bata Date: Mon, 19 Aug 2024 19:52:14 +0900 Subject: [PATCH 4/4] Print erros on browser console --- e2e_tests/test_dashboard/visual_regression_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_tests/test_dashboard/visual_regression_test.py b/e2e_tests/test_dashboard/visual_regression_test.py index 7bf61356..3d38d6de 100644 --- a/e2e_tests/test_dashboard/visual_regression_test.py +++ b/e2e_tests/test_dashboard/visual_regression_test.py @@ -193,11 +193,11 @@ def test_study_analytics( run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: study = run_study(storage) - study_id = study._study_id study_name = study.study_name url = f"{server_url}/studies/{study_id}" + page.on("console", lambda msg: print(f"error: {msg.text}") if msg.type == "error" else None) page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/analytics']")