From 8a8cd29f21410a65f3c577c43fc27e77ed1b12af Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 29 Sep 2023 12:34:35 +0900 Subject: [PATCH 1/4] Do exhaustive evaluation of acquisition function when the search space is small enough --- optuna_dashboard/preferential/samplers/gp.py | 47 ++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index d023e741..f4415a38 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -5,6 +5,7 @@ import math from typing import Any from typing import Callable from typing import cast +import warnings import botorch.acquisition.analytic import botorch.models.model @@ -17,6 +18,8 @@ import numpy as np import optuna import optuna._transform from optuna.distributions import CategoricalDistribution +from optuna.distributions import FloatDistribution +from optuna.distributions import IntDistribution import torch from torch import Tensor @@ -359,11 +362,45 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): ) # TODO: Make it possible to apply it on mixed search space - if all(isinstance(dist, CategoricalDistribution) for dist in search_space.values()): + def get_all_possible_params(dist: optuna.distributions.BaseDistribution) -> list[Any]: + if isinstance(dist, CategoricalDistribution): + return dist.choices + elif isinstance(dist, (IntDistribution, FloatDistribution)): + return list(np.arange(dist.low, dist.high, dist.step)) + else: + return [] + + all_possible_params = { + name: get_all_possible_params(dist) for name, dist in search_space.items() + } + + has_categorical = any( + isinstance(dist, CategoricalDistribution) for dist in search_space.values() + ) + is_all_discrete = all( + len(possible_params) > 0 for possible_params in all_possible_params.values() + ) + can_evaluate_all = ( + is_all_discrete + and np.prod( + [len(possible_params) for possible_params in all_possible_params.values()] + ) + <= 1e6 + ) + print(has_categorical, is_all_discrete, can_evaluate_all) + + if has_categorical and not can_evaluate_all: + warnings.warn( + "The objective function has categorical parameters, " + "but the total search space is too large to be enumerated. " + "This may result in significantly bad performance." + ) + + if is_all_discrete and can_evaluate_all: all_param_combinations = itertools.product( *[ - [(name, choice) for choice in cast(CategoricalDistribution, dist).choices] - for name, dist in search_space.items() + [(name, choice) for choice in possible_params] + for name, possible_params in all_possible_params.items() ] ) choices = torch.tensor( @@ -395,6 +432,10 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): param_name: str, param_distribution: optuna.distributions.BaseDistribution, ) -> Any: + warnings.warn( + f"Dynamic search space detected. Falling back to {self.independent_sampler}." + ) + return self.independent_sampler.sample_independent( study, trial, param_name, param_distribution ) From 5a4f1ed195f3c9d4f4a724defaa4b23bce951e5a Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 29 Sep 2023 12:40:30 +0900 Subject: [PATCH 2/4] Fix linter --- optuna_dashboard/preferential/samplers/gp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index f4415a38..7e5e2c35 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -4,7 +4,6 @@ import itertools import math from typing import Any from typing import Callable -from typing import cast import warnings import botorch.acquisition.analytic From 483b49bb9755c51c14230220f2948a0359b4b212 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 29 Sep 2023 12:42:43 +0900 Subject: [PATCH 3/4] Fix mypy --- optuna_dashboard/preferential/samplers/gp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 7e5e2c35..44427d9b 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -363,7 +363,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): # TODO: Make it possible to apply it on mixed search space def get_all_possible_params(dist: optuna.distributions.BaseDistribution) -> list[Any]: if isinstance(dist, CategoricalDistribution): - return dist.choices + return list(dist.choices) elif isinstance(dist, (IntDistribution, FloatDistribution)): return list(np.arange(dist.low, dist.high, dist.step)) else: From 87c41cbf0d1f40405a80a5ced84142165cf502fa Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 29 Sep 2023 17:37:49 +0900 Subject: [PATCH 4/4] Fix warning message --- optuna_dashboard/preferential/samplers/gp.py | 44 ++++++++++++-------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 44427d9b..cb03fb8f 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -373,27 +373,36 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): name: get_all_possible_params(dist) for name, dist in search_space.items() } - has_categorical = any( - isinstance(dist, CategoricalDistribution) for dist in search_space.values() - ) is_all_discrete = all( len(possible_params) > 0 for possible_params in all_possible_params.values() ) - can_evaluate_all = ( - is_all_discrete - and np.prod( - [len(possible_params) for possible_params in all_possible_params.values()] - ) - <= 1e6 + search_space_size = np.prod( + [len(possible_params) for possible_params in all_possible_params.values()] ) - print(has_categorical, is_all_discrete, can_evaluate_all) + # TODO(contramundum53): Fix this arbitrarily chosen limit. + size_limit = 1e6 + can_evaluate_all = is_all_discrete and search_space_size <= size_limit - if has_categorical and not can_evaluate_all: - warnings.warn( - "The objective function has categorical parameters, " - "but the total search space is too large to be enumerated. " - "This may result in significantly bad performance." - ) + if ( + any(isinstance(dist, CategoricalDistribution) for dist in search_space.values()) + and not can_evaluate_all + ): + if is_all_discrete: + warnings.warn( + "The objective function has categorical parameters, " + "but the total search space is too large to be enumerated. " + f"(Search space size: {search_space_size} > limit: {size_limit})" + "This may result in significantly bad performance." + ) + else: + warnings.warn( + "The objective function has categorical parameters, " + "but the search space cannot be enumerated because " + "it also contains continuous parameters. " + "This may result in significantly bad performance. " + "You can work around this problem by specifying 'step' " + "in each continuous parameter." + ) if is_all_discrete and can_evaluate_all: all_param_combinations = itertools.product( @@ -432,7 +441,8 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): param_distribution: optuna.distributions.BaseDistribution, ) -> Any: warnings.warn( - f"Dynamic search space detected. Falling back to {self.independent_sampler}." + "Dynamic search space detected. " + f"Falling back to {self.independent_sampler.__class__.__name__}." ) return self.independent_sampler.sample_independent(