diff --git a/.github/workflows/e2e-dashboard-tests.yml b/.github/workflows/e2e-dashboard-tests.yml index 0f293273..71d77f0e 100644 --- a/.github/workflows/e2e-dashboard-tests.yml +++ b/.github/workflows/e2e-dashboard-tests.yml @@ -6,8 +6,12 @@ on: paths: - '.github/workflows/e2e-dashboard-tests.yml' - '**.py' - - '**.ts' - - '**.tsx' + - 'tslib/**.ts' + - 'tslib/**.tsx' + - 'tslib/**/package.json' + - 'tslib/**/package-lock.json' + - 'optuna_dashboard/**.ts' + - 'optuna_dashboard/**.tsx' - 'optuna_dashboard/package.json' - 'optuna_dashboard/package-lock.json' - 'optuna_dashboard/tsconfig.json' @@ -37,7 +41,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.11' architecture: x64 - name: Setup Optuna ${{ matrix.optuna-version }} diff --git a/.github/workflows/e2e-standalone-tests.yml b/.github/workflows/e2e-standalone-tests.yml index d449e2b7..a599336b 100644 --- a/.github/workflows/e2e-standalone-tests.yml +++ b/.github/workflows/e2e-standalone-tests.yml @@ -5,6 +5,10 @@ on: - main paths: - '.github/workflows/e2e-standalone-tests.yml' + - 'tslib/**.ts' + - 'tslib/**.tsx' + - 'tslib/**/package.json' + - 'tslib/**/package-lock.json' - 'standalone_app/**.ts' - 'standalone_app/**.tsx' - 'standalone_app/package.json' @@ -34,7 +38,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.11' architecture: x64 - name: Setup Optuna ${{ matrix.optuna-version }} diff --git a/e2e_tests/test_dashboard/visual_regression_test.py b/e2e_tests/test_dashboard/visual_regression_test.py index ea6ff930..3d38d6de 100644 --- a/e2e_tests/test_dashboard/visual_regression_test.py +++ b/e2e_tests/test_dashboard/visual_regression_test.py @@ -27,7 +27,7 @@ def run_single_objective_study(storage: optuna.storages.InMemoryStorage) -> optu x2 = trial.suggest_float("x2", 0, 10) return (x1 - 2) ** 2 + (x2 - 5) ** 2 - study.optimize(objective, n_trials=50) + study.optimize(objective, n_trials=20) return study @@ -46,20 +46,6 @@ def run_single_trial_objective_study(storage: optuna.storages.InMemoryStorage) - return study -def run_single_1param_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study( - study_name="single-1-param", storage=storage, direction="maximize", sampler=sampler - ) - - def objective(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - return -((x1 - 2) ** 2) - - study.optimize(objective, n_trials=50) - return study - - def run_single_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: # Single-objective study with dynamic search space sampler = optuna.samplers.RandomSampler(seed=0) @@ -78,24 +64,6 @@ def run_single_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) return study -def run_single_inf_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: - # Single objective study with 'inf', '-inf', or 'nan' value - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study(study_name="single-inf", storage=storage, sampler=sampler) - - def objective(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - return float("inf") - elif trial.number % 3 == 1: - return float("-inf") - else: - return x**2 - - study.optimize(objective, n_trials=50) - return study - - def run_multi_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: # Multi-objective study sampler = optuna.samplers.RandomSampler(seed=0) @@ -113,7 +81,7 @@ def run_multi_objective_study(storage: optuna.storages.InMemoryStorage) -> optun v1 = (x - 5) ** 2 + (y - 5) ** 2 return v0, v1 - study.optimize(objective, n_trials=50) + study.optimize(objective, n_trials=20) return study @@ -142,7 +110,7 @@ def run_multi_dynamic_objective_study(storage: optuna.storages.InMemoryStorage) v1 = (x - 2) ** 2 + (y - 3) ** 2 return v0, v1 - study.optimize(objective, n_trials=50) + study.optimize(objective, n_trials=20) return study @@ -167,66 +135,6 @@ def run_single_pruned_without_report_objective_study( return study -def run_single_inf_report_objective_study( - storage: optuna.storages.InMemoryStorage, -) -> optuna.Study: - # Single objective pruned after reported 'inf', '-inf', or 'nan' - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study(study_name="single-inf-report", storage=storage, sampler=sampler) - - def objective(trial: optuna.Trial) -> float: - x = trial.suggest_float("x", -10, 10) - if trial.number % 3 == 0: - trial.report(float("inf"), 1) - elif trial.number % 3 == 1: - trial.report(float("-inf"), 1) - else: - trial.report(float("nan"), 1) - - if x > 0: - raise optuna.TrialPruned() - else: - return x**2 - - study.optimize(objective, n_trials=50) - return study - - -def run_issue_410_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: - # Issue 410 - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study(study_name="issue-410", storage=storage, sampler=sampler) - - def objective(trial: optuna.Trial) -> float: - trial.suggest_categorical("resample_rate", ["50ms"]) - trial.suggest_categorical("channels", ["all"]) - trial.suggest_categorical("window_size", [256]) - if trial.number > 15: - raise Exception("Unexpected error") - trial.suggest_categorical("cbow", [True]) - trial.suggest_categorical("model", ["m1"]) - - trial.set_user_attr("epochs", 0) - trial.set_user_attr("deterministic", True) - if trial.number > 10: - raise Exception("unexpeccted error") - trial.set_user_attr("folder", "/path/to/folder") - trial.set_user_attr("resample_type", "foo") - trial.set_user_attr("run_id", "0001") - return 1.0 - - study.optimize(objective, n_trials=20, catch=(Exception,)) - return study - - -def run_single_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: - # No trials single-objective study - sampler = optuna.samplers.RandomSampler(seed=0) - study = optuna.create_study(study_name="single-no-trials", storage=storage, sampler=sampler) - - return study - - def run_multi_no_trials_objective_study(storage: optuna.storages.InMemoryStorage) -> optuna.Study: # No trials multi-objective study sampler = optuna.samplers.RandomSampler(seed=0) @@ -244,15 +152,10 @@ parameterize_studies = pytest.mark.parametrize( [ run_single_objective_study, run_single_trial_objective_study, - run_single_1param_objective_study, run_single_dynamic_objective_study, - run_single_inf_objective_study, run_multi_objective_study, run_multi_dynamic_objective_study, run_single_pruned_without_report_objective_study, - run_single_inf_report_objective_study, - run_issue_410_objective_study, - run_single_no_trials_objective_study, run_multi_no_trials_objective_study, ], ) @@ -273,6 +176,7 @@ def test_study_list( page.goto(server_url) page.click(f"a[href='/dashboard/studies/{study_id}']") + page.wait_for_selector(".MuiTypography-body1") element = page.query_selector(".MuiTypography-body1") assert element is not None @@ -289,14 +193,15 @@ def test_study_analytics( run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: study = run_study(storage) - study_id = study._study_id study_name = study.study_name url = f"{server_url}/studies/{study_id}" + page.on("console", lambda msg: print(f"error: {msg.text}") if msg.type == "error" else None) page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/analytics']") + page.wait_for_selector(".MuiTypography-body1", timeout=60 * 1000) element = page.query_selector(".MuiTypography-body1") assert element is not None @@ -321,6 +226,7 @@ def test_trial_list( page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/trials']") + page.wait_for_selector(".MuiTypography-body1") element = page.query_selector(".MuiTypography-body1") assert element is not None @@ -345,6 +251,7 @@ def test_trial_table( page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/trialTable']") + page.wait_for_selector(".MuiTypography-body1") element = page.query_selector(".MuiTypography-body1") assert element is not None @@ -367,6 +274,7 @@ def test_trial_note( url = f"{server_url}/studies/{study_id}" page.goto(url) + page.wait_for_selector(".MuiTypography-body1") page.click(f"a[href='/dashboard/studies/{study_id}/note']") element = page.query_selector(".MuiTypography-body1") diff --git a/tslib/react/test/generate_assets.py b/tslib/react/test/generate_assets.py index f7bdc33f..0390bfd1 100644 --- a/tslib/react/test/generate_assets.py +++ b/tslib/react/test/generate_assets.py @@ -447,6 +447,34 @@ def create_optuna_storage( trial.report(trial.number, step=0) trial.report(trial.number + 1, step=1) + # optuna-dashboard issue 410 + # https://github.com/optuna/optuna-dashboard/issues/410 + study = optuna.create_study( + study_name="optuna-dashboard-issue-410", + storage=storage, + sampler=optuna.samplers.RandomSampler(), + ) + + def objective_issue_410(trial: optuna.Trial) -> float: + trial.suggest_categorical("resample_rate", ["50ms"]) + trial.suggest_categorical("channels", ["all"]) + trial.suggest_categorical("window_size", [256]) + if trial.number > 15: + raise Exception("Unexpected error") + trial.suggest_categorical("cbow", [True]) + trial.suggest_categorical("model", ["m1"]) + + trial.set_user_attr("epochs", 0) + trial.set_user_attr("deterministic", True) + if trial.number > 10: + raise Exception("unexpeccted error") + trial.set_user_attr("folder", "/path/to/folder") + trial.set_user_attr("resample_type", "foo") + trial.set_user_attr("run_id", "0001") + return 1.0 + + study.optimize(objective_issue_410, n_trials=20, catch=(Exception,)) + def main() -> None: remove_assets()