mirror of
https://github.com/wassname/optuna-dashboard.git
synced 2026-09-06 17:00:30 +08:00
491 lines
17 KiB
Python
491 lines
17 KiB
Python
import json
|
|
import logging
|
|
import math
|
|
import os.path
|
|
import shutil
|
|
from typing import Tuple
|
|
|
|
import optuna
|
|
from optuna.distributions import CategoricalDistribution
|
|
from optuna.distributions import FloatDistribution
|
|
from optuna.importance import get_param_importances
|
|
from optuna.importance import PedAnovaImportanceEvaluator
|
|
from optuna.storages import BaseStorage
|
|
from optuna.storages import JournalFileStorage
|
|
from optuna.storages import JournalStorage
|
|
|
|
|
|
optuna.logging.set_verbosity(logging.CRITICAL)
|
|
BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "asset")
|
|
|
|
|
|
def remove_assets() -> None:
|
|
if os.path.exists(BASE_DIR):
|
|
shutil.rmtree(BASE_DIR)
|
|
os.mkdir(BASE_DIR)
|
|
|
|
|
|
def create_optuna_storage(
|
|
storage: BaseStorage, params_importances: dict[str, list[dict[str, float]]]
|
|
) -> optuna.storages.InMemoryStorage:
|
|
# Single-objective study
|
|
study = optuna.create_study(
|
|
study_name="single-objective", storage=storage, sampler=optuna.samplers.RandomSampler()
|
|
)
|
|
|
|
def objective_single(trial: optuna.Trial) -> float:
|
|
x1 = trial.suggest_float("x1", 0, 10)
|
|
x2 = trial.suggest_float("x2", 0, 10)
|
|
trial.suggest_categorical("x3", ["foo", "bar"])
|
|
return (x1 - 2) ** 2 + (x2 - 5) ** 2
|
|
|
|
study.optimize(objective_single, n_trials=100)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Single-objective study with dynamic search space
|
|
study = optuna.create_study(
|
|
study_name="single-objective-dynamic", storage=storage, direction="maximize"
|
|
)
|
|
|
|
def objective_single_dynamic(trial: optuna.Trial) -> float:
|
|
category = trial.suggest_categorical("category", ["foo", "bar"])
|
|
if category == "foo":
|
|
return (trial.suggest_float("x1", 0, 10) - 2) ** 2
|
|
else:
|
|
return -((trial.suggest_float("x2", -10, 0) + 5) ** 2)
|
|
|
|
study.optimize(objective_single_dynamic, n_trials=50)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
study = optuna.create_study(
|
|
study_name="check-rank-plot", storage=storage, sampler=optuna.samplers.RandomSampler()
|
|
)
|
|
|
|
def objective_single(trial: optuna.Trial) -> float:
|
|
x1 = trial.suggest_float("x1", 0, 10)
|
|
x2 = trial.suggest_float("x2", 0, 10)
|
|
trial.suggest_float("x3", 0, 10)
|
|
trial.suggest_float("x4", 0, 10)
|
|
trial.suggest_float("x5", 0, 10)
|
|
trial.suggest_float("x6", 0, 10)
|
|
return (x1 - 2) ** 2 + (x2 - 5) ** 2
|
|
|
|
study.optimize(objective_single, n_trials=1000)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Single-objective study
|
|
study = optuna.create_study(study_name="single-objective-user-attrs", storage=storage)
|
|
|
|
def objective_single_user_attr(trial: optuna.Trial) -> float:
|
|
x1 = trial.suggest_float("x1", 0, 10)
|
|
x2 = trial.suggest_float("x2", 0, 10)
|
|
if x1 < 5:
|
|
trial.set_user_attr("X", "foo")
|
|
else:
|
|
trial.set_user_attr("X", "bar")
|
|
trial.set_user_attr("Y", x1 + x2)
|
|
return (x1 - 2) ** 2 + (x2 - 5) ** 2
|
|
|
|
study.optimize(objective_single_user_attr, n_trials=100)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Single objective study with 'inf', '-inf', or 'nan' value
|
|
study = optuna.create_study(study_name="single-inf", storage=storage)
|
|
|
|
def objective_single_inf(trial: optuna.Trial) -> float:
|
|
x = trial.suggest_float("x", -10, 10)
|
|
if trial.number % 3 == 0:
|
|
return float("inf")
|
|
elif trial.number % 3 == 1:
|
|
return float("-inf")
|
|
else:
|
|
return x**2
|
|
|
|
study.optimize(objective_single_inf, n_trials=50)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Single objective pruned after reported 'inf', '-inf', or 'nan'
|
|
study = optuna.create_study(study_name="single-inf-report", storage=storage)
|
|
|
|
def objective_single_inf_report(trial: optuna.Trial) -> float:
|
|
x = trial.suggest_float("x", -10, 10)
|
|
if trial.number % 3 == 0:
|
|
trial.report(float("inf"), 1)
|
|
elif trial.number % 3 == 1:
|
|
trial.report(float("-inf"), 1)
|
|
else:
|
|
trial.report(float("nan"), 1)
|
|
|
|
if x > 0:
|
|
raise optuna.TrialPruned()
|
|
else:
|
|
return x**2
|
|
|
|
study.optimize(objective_single_inf_report, n_trials=50)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Single objective with reported nan value
|
|
study = optuna.create_study(study_name="single-nan-report", storage=storage)
|
|
|
|
def objective_single_nan_report(trial: optuna.Trial) -> float:
|
|
x1 = trial.suggest_float("x1", 0, 10)
|
|
x2 = trial.suggest_float("x2", 0, 10)
|
|
trial.report(0.5, step=0)
|
|
trial.report(math.nan, step=1)
|
|
return (x1 - 2) ** 2 + (x2 - 5) ** 2
|
|
|
|
study.optimize(objective_single_nan_report, n_trials=100)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Single-objective study with 1 parameter
|
|
study = optuna.create_study(
|
|
study_name="single-objective-1-param", storage=storage, direction="maximize"
|
|
)
|
|
|
|
def objective_single_with_1param(trial: optuna.Trial) -> float:
|
|
x1 = trial.suggest_float("x1", 0, 10)
|
|
return -((x1 - 2) ** 2)
|
|
|
|
study.optimize(objective_single_with_1param, n_trials=50)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Single-objective study with 1 parameter
|
|
study = optuna.create_study(study_name="long-parameter-names", storage=storage)
|
|
|
|
def objective_long_parameter_names(trial: optuna.Trial) -> float:
|
|
x1 = trial.suggest_float(
|
|
"x1_long_parameter_names_long_long_long_long_long_long_long_long_long_long", 0, 10
|
|
)
|
|
x2 = trial.suggest_float(
|
|
"x2_long_parameter_names_long_long_long_long_long_long_long_long_long_long", 0, 10
|
|
)
|
|
return (x1 - 2) ** 2 + (x2 - 5) ** 2
|
|
|
|
study.optimize(objective_long_parameter_names, n_trials=50)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Multi-objective study
|
|
study = optuna.create_study(
|
|
study_name="multi-objective",
|
|
storage=storage,
|
|
directions=["minimize", "minimize"],
|
|
)
|
|
study.set_metric_names(["v0", "v1"])
|
|
|
|
def objective_multi(trial: optuna.Trial) -> Tuple[float, float]:
|
|
x = trial.suggest_float("x", 0, 5)
|
|
y = trial.suggest_float("y", 0, 3)
|
|
v0 = 4 * x**2 + 4 * y**2
|
|
v1 = (x - 5) ** 2 + (y - 5) ** 2
|
|
return v0, v1
|
|
|
|
study.optimize(objective_multi, n_trials=50)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Multi-objective study with dynamic search space
|
|
study = optuna.create_study(
|
|
study_name="multi-dynamic", storage=storage, directions=["minimize", "minimize"]
|
|
)
|
|
|
|
def objective_multi_dynamic(trial: optuna.Trial) -> Tuple[float, float]:
|
|
category = trial.suggest_categorical("category", ["foo", "bar"])
|
|
if category == "foo":
|
|
x = trial.suggest_float("x1", 0, 5)
|
|
y = trial.suggest_float("y1", 0, 3)
|
|
v0 = 4 * x**2 + 4 * y**2
|
|
v1 = (x - 5) ** 2 + (y - 5) ** 2
|
|
return v0, v1
|
|
else:
|
|
x = trial.suggest_float("x2", 0, 5)
|
|
y = trial.suggest_float("y2", 0, 3)
|
|
v0 = 2 * x**2 + 2 * y**2
|
|
v1 = (x - 2) ** 2 + (y - 3) ** 2
|
|
return v0, v1
|
|
|
|
study.optimize(objective_multi_dynamic, n_trials=50)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Pruning with no intermediate values
|
|
study = optuna.create_study(study_name="binh-korn-function-with-constraints", storage=storage)
|
|
|
|
def objective_prune_with_no_trials(trial: optuna.Trial) -> float:
|
|
x = trial.suggest_float("x", -15, 30)
|
|
y = trial.suggest_float("y", -15, 30)
|
|
v = x**2 + y**2
|
|
if v > 100:
|
|
raise optuna.TrialPruned()
|
|
return v
|
|
|
|
study.optimize(objective_prune_with_no_trials, n_trials=100)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# With failed trials
|
|
study = optuna.create_study(study_name="failed trials", storage=storage)
|
|
|
|
def objective_sometimes_got_failed(trial: optuna.Trial) -> float:
|
|
x = trial.suggest_float("x", -15, 30)
|
|
y = trial.suggest_float("y", -15, 30)
|
|
v = x**2 + y**2
|
|
if v > 100:
|
|
raise ValueError("unexpected error")
|
|
return v
|
|
|
|
study.optimize(objective_sometimes_got_failed, n_trials=100, catch=(Exception,))
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# No trials single-objective study
|
|
study = optuna.create_study(study_name="no trials single-objective study", storage=storage)
|
|
study.set_user_attr("foo", "bar")
|
|
|
|
# study with waiting trials
|
|
study = optuna.create_study(study_name="waiting-trials", storage=storage)
|
|
study.enqueue_trial({"x": 0, "y": 10})
|
|
study.enqueue_trial({"x": 10, "y": 20})
|
|
|
|
# Study with Running Trials
|
|
study = optuna.create_study(
|
|
study_name="running-trials", storage=storage, directions=["minimize", "maximize"]
|
|
)
|
|
study.set_metric_names(["auc", "val_loss"])
|
|
study.enqueue_trial({"x": 10, "y": "Foo"})
|
|
study.ask({"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])})
|
|
study.ask({"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])})
|
|
|
|
# Single-objective study with constraints
|
|
def constraints(trial: optuna.Trial) -> list[float]:
|
|
return trial.user_attrs["constraint"]
|
|
|
|
study = optuna.create_study(
|
|
study_name="A single objective constraint optimization study",
|
|
storage=storage,
|
|
sampler=optuna.samplers.TPESampler(constraints_func=constraints),
|
|
)
|
|
|
|
def objective_constraints(trial: optuna.Trial) -> float:
|
|
x = trial.suggest_float("x", -15, 30)
|
|
y = trial.suggest_float("y", -15, 30)
|
|
v0 = 4 * x**2 + 4 * y**2
|
|
trial.set_user_attr("constraint", [1000 - v0, x - 10, y - 10])
|
|
return v0
|
|
|
|
study.optimize(objective_constraints, n_trials=100)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
|
|
# Study with Running Trials
|
|
study = optuna.create_study(
|
|
study_name="objective-form-widgets",
|
|
storage=storage,
|
|
directions=["minimize", "minimize", "minimize", "minimize"],
|
|
)
|
|
study.set_metric_names(
|
|
["Slider Objective", "Good or Bad", "Text Input Objective", "Validation Loss"]
|
|
)
|
|
trial = study.ask(
|
|
{"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])}
|
|
)
|
|
trial.set_user_attr("val_loss", 0.2)
|
|
study.ask({"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])})
|
|
trial.set_user_attr("val_loss", 0.5)
|
|
|
|
# No trials multi-objective study
|
|
optuna.create_study(
|
|
study_name="no trials multi-objective study",
|
|
storage=storage,
|
|
directions=["minimize", "maximize"],
|
|
)
|
|
|
|
# Single-objective study with intermediate values
|
|
study = optuna.create_study(study_name="intermediate-values", storage=storage)
|
|
|
|
def objective_intermediate_values(trial: optuna.Trial) -> float:
|
|
trial.report(trial.number, step=0)
|
|
trial.report(trial.number + 1, step=1)
|
|
return 0.0
|
|
|
|
study.optimize(objective_intermediate_values, n_trials=10)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
trial = study.ask(
|
|
{"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])}
|
|
) # To create a running trial
|
|
trial.report(trial.number, step=0)
|
|
trial.report(trial.number + 1, step=1)
|
|
|
|
# Single-objective study with intermediate values and constraints
|
|
def constraints(trial: optuna.Trial) -> list[float]:
|
|
return trial.user_attrs["constraint"]
|
|
|
|
study = optuna.create_study(
|
|
study_name="intermediate-values-constraints",
|
|
storage=storage,
|
|
sampler=optuna.samplers.NSGAIISampler(constraints_func=constraints),
|
|
)
|
|
|
|
def objective_intermediate_values_constraints(trial: optuna.Trial) -> float:
|
|
trial.set_user_attr("constraint", [trial.number % 2])
|
|
|
|
trial.report(trial.number, step=0)
|
|
trial.report(trial.number + 1, step=1)
|
|
return 0.0
|
|
|
|
study.optimize(objective_intermediate_values_constraints, n_trials=10)
|
|
params_importances[study.study_name] = [
|
|
get_param_importances(
|
|
study,
|
|
target=lambda trial: trial.values[objective_id],
|
|
evaluator=PedAnovaImportanceEvaluator(),
|
|
)
|
|
for objective_id in range(len(study.directions))
|
|
]
|
|
trial = study.ask(
|
|
{"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])}
|
|
) # To create a running trial
|
|
trial.report(trial.number, step=0)
|
|
trial.report(trial.number + 1, step=1)
|
|
|
|
# optuna-dashboard issue 410
|
|
# https://github.com/optuna/optuna-dashboard/issues/410
|
|
study = optuna.create_study(
|
|
study_name="optuna-dashboard-issue-410",
|
|
storage=storage,
|
|
sampler=optuna.samplers.RandomSampler(),
|
|
)
|
|
|
|
def objective_issue_410(trial: optuna.Trial) -> float:
|
|
trial.suggest_categorical("resample_rate", ["50ms"])
|
|
trial.suggest_categorical("channels", ["all"])
|
|
trial.suggest_categorical("window_size", [256])
|
|
if trial.number > 15:
|
|
raise Exception("Unexpected error")
|
|
trial.suggest_categorical("cbow", [True])
|
|
trial.suggest_categorical("model", ["m1"])
|
|
|
|
trial.set_user_attr("epochs", 0)
|
|
trial.set_user_attr("deterministic", True)
|
|
if trial.number > 10:
|
|
raise Exception("unexpeccted error")
|
|
trial.set_user_attr("folder", "/path/to/folder")
|
|
trial.set_user_attr("resample_type", "foo")
|
|
trial.set_user_attr("run_id", "0001")
|
|
return 1.0
|
|
|
|
study.optimize(objective_issue_410, n_trials=20, catch=(Exception,))
|
|
|
|
|
|
def main() -> None:
|
|
remove_assets()
|
|
storage = JournalStorage(JournalFileStorage(os.path.join(BASE_DIR, "journal.log")))
|
|
params_importances: dict[str, dict[str, float]] = {}
|
|
create_optuna_storage(storage, params_importances)
|
|
|
|
with open(os.path.join(BASE_DIR, "params_importances.json"), "w") as f:
|
|
json.dump(params_importances, f, indent=2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|