diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 140ad8b1..8183f48d 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -37,6 +37,7 @@ from packaging import version from . import _note as note from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache +from ._pareto_front import get_pareto_front_trials from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -308,6 +309,12 @@ def create_app(storage: BaseStorage, debug: bool = False) -> Bottle: response.status = 404 # Not found return {"reason": f"study_id={study_id} is not found"} trials = get_trials(storage, study_id) + + # TODO(c-bata): Cache best_trials + if summary.directions == 1: + best_trials = [storage.get_best_trial(study_id)] + else: + best_trials = get_pareto_front_trials(trials=trials, directions=summary.directions) ( # TODO: intersection_search_space and union_search_space look more clear since now we # have union_user_attrs. @@ -318,6 +325,7 @@ def create_app(storage: BaseStorage, debug: bool = False) -> Bottle: ) = get_cached_extra_study_property(study_id, trials) return serialize_study_detail( summary, + best_trials, trials[after:], intersection, union, diff --git a/optuna_dashboard/_pareto_front.py b/optuna_dashboard/_pareto_front.py new file mode 100644 index 00000000..f6af6831 --- /dev/null +++ b/optuna_dashboard/_pareto_front.py @@ -0,0 +1,109 @@ +# This code is taken from the below to keep support for older versions. +# https://github.com/optuna/optuna/blob/v3.0.5/optuna/study/_multi_objective.py + +from __future__ import annotations +from typing import Optional +from typing import Sequence + +from optuna.study._study_direction import StudyDirection +from optuna.trial import FrozenTrial +from optuna.trial import TrialState + + +def get_pareto_front_trials( + trials: Sequence[FrozenTrial], directions: Sequence[StudyDirection] +) -> list[FrozenTrial]: + if len(directions) == 2: + return _get_pareto_front_trials_2d(trials, directions) # Log-linear in number of trials. + return _get_pareto_front_trials_nd(trials, directions) # Quadratic in number of trials. + + +def _get_pareto_front_trials_2d( + trials: Sequence[FrozenTrial], directions: Sequence[StudyDirection] +) -> list[FrozenTrial]: + trials = [trial for trial in trials if trial.state == TrialState.COMPLETE] + + n_trials = len(trials) + if n_trials == 0: + return [] + + trials.sort( + key=lambda trial: ( + _normalize_value(trial.values[0], directions[0]), + _normalize_value(trial.values[1], directions[1]), + ), + ) + + last_nondominated_trial = trials[0] + pareto_front = [last_nondominated_trial] + for i in range(1, n_trials): + trial = trials[i] + if _dominates(last_nondominated_trial, trial, directions): + continue + pareto_front.append(trial) + last_nondominated_trial = trial + + pareto_front.sort(key=lambda trial: trial.number) + return pareto_front + + +def _get_pareto_front_trials_nd( + trials: Sequence[FrozenTrial], directions: Sequence[StudyDirection] +) -> list[FrozenTrial]: + pareto_front = [] + trials = [t for t in trials if t.state == TrialState.COMPLETE] + + # TODO(vincent): Optimize (use the fast non dominated sort defined in the NSGA-II paper). + for trial in trials: + dominated = False + for other in trials: + if _dominates(other, trial, directions): + dominated = True + break + + if not dominated: + pareto_front.append(trial) + + return pareto_front + + +def _dominates( + trial0: FrozenTrial, trial1: FrozenTrial, directions: Sequence[StudyDirection] +) -> bool: + values0 = trial0.values + values1 = trial1.values + + assert values0 is not None + assert values1 is not None + + if len(values0) != len(values1): + raise ValueError("Trials with different numbers of objectives cannot be compared.") + + if len(values0) != len(directions): + raise ValueError( + "The number of the values and the number of the objectives are mismatched." + ) + + if trial0.state != TrialState.COMPLETE: + return False + + if trial1.state != TrialState.COMPLETE: + return True + + normalized_values0 = [_normalize_value(v, d) for v, d in zip(values0, directions)] + normalized_values1 = [_normalize_value(v, d) for v, d in zip(values1, directions)] + + if normalized_values0 == normalized_values1: + return False + + return all(v0 <= v1 for v0, v1 in zip(normalized_values0, normalized_values1)) + + +def _normalize_value(value: Optional[float], direction: StudyDirection) -> float: + if value is None: + value = float("inf") + + if direction is StudyDirection.MAXIMIZE: + value = -value + + return value diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 9f42e83d..d0c58622 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -73,6 +73,7 @@ def serialize_study_summary(summary: StudySummary) -> dict[str, Any]: def serialize_study_detail( summary: StudySummary, + best_trials: list[FrozenTrial], trials: list[FrozenTrial], intersection: list[tuple[str, BaseDistribution]], union: list[tuple[str, BaseDistribution]], @@ -87,6 +88,7 @@ def serialize_study_detail( serialized["datetime_start"] = summary.datetime_start.isoformat() serialized["trials"] = [serialize_frozen_trial(summary._study_id, trial) for trial in trials] + serialized["best_trials"] = [serialize_frozen_trial(summary._study_id, trial) for trial in best_trials] serialized["intersection_search_space"] = serialize_search_space(intersection) serialized["union_search_space"] = serialize_search_space(union) serialized["union_user_attrs"] = [{"key": a[0], "sortable": a[1]} for a in union_user_attrs]