From a4f2dd2138658cc9aef64ffe73877f82cb2a4856 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Wed, 23 Dec 2020 18:27:16 +0100 Subject: [PATCH] [Tune]Add integer loguniform support (#12994) * Add integer quantization and loguniform support * Fix hyperopt qloguniform not being np.log'd first * Add tests, __init__ * Try to fix tests, better exceptions * Tweak docstrings * Type checks in SearchSpaceTest * Update docs * Lint, tests * Update doc/source/tune/api_docs/search_space.rst Co-authored-by: Kai Fricke Co-authored-by: Kai Fricke --- doc/source/tune/api_docs/search_space.rst | 10 +++- python/ray/tune/__init__.py | 12 ++--- python/ray/tune/sample.py | 59 +++++++++++++++++++++-- python/ray/tune/suggest/bohb.py | 10 +++- python/ray/tune/suggest/hyperopt.py | 24 ++++++--- python/ray/tune/suggest/nevergrad.py | 15 ++++-- python/ray/tune/suggest/skopt.py | 31 ++++++------ python/ray/tune/tests/test_sample.py | 26 ++++++++-- 8 files changed, 147 insertions(+), 40 deletions(-) diff --git a/doc/source/tune/api_docs/search_space.rst b/doc/source/tune/api_docs/search_space.rst index 3c069760f..9e5014031 100644 --- a/doc/source/tune/api_docs/search_space.rst +++ b/doc/source/tune/api_docs/search_space.rst @@ -192,10 +192,18 @@ For a high-level overview, see this example: # Sample a integer uniformly between -9 (inclusive) and 15 (exclusive) "randint": tune.randint(-9, 15), + # Sample a integer uniformly between 1 (inclusive) and 10 (exclusive), + # while sampling in log space + "lograndint": tune.lograndint(1, 10), + # Sample a random uniformly between -21 (inclusive) and 12 (inclusive (!)) # rounding to increments of 3 (includes 12) "qrandint": tune.qrandint(-21, 12, 3), + # Sample a integer uniformly between 1 (inclusive) and 10 (inclusive (!)), + # while sampling in log space and rounding to increments of 2 + "qlograndint": tune.qlograndint(1, 10, 2), + # Sample an option uniformly from the specified choices "choice": tune.choice(["a", "b", "c"]), @@ -266,4 +274,4 @@ Grid Search API References ---------- -See also :ref:`tune-basicvariant`. \ No newline at end of file +See also :ref:`tune-basicvariant`. diff --git a/python/ray/tune/__init__.py b/python/ray/tune/__init__.py index 8171b0756..58906af40 100644 --- a/python/ray/tune/__init__.py +++ b/python/ray/tune/__init__.py @@ -16,8 +16,8 @@ from ray.tune.session import ( from ray.tune.progress_reporter import (ProgressReporter, CLIReporter, JupyterNotebookReporter) from ray.tune.sample import (function, sample_from, uniform, quniform, choice, - randint, qrandint, randn, qrandn, loguniform, - qloguniform) + randint, lograndint, qrandint, qlograndint, randn, + qrandn, loguniform, qloguniform) from ray.tune.suggest import create_searcher from ray.tune.schedulers import create_scheduler @@ -26,10 +26,10 @@ __all__ = [ "register_env", "register_trainable", "run", "run_experiments", "with_parameters", "Stopper", "EarlyStopping", "Experiment", "function", "sample_from", "track", "uniform", "quniform", "choice", "randint", - "qrandint", "randn", "qrandn", "loguniform", "qloguniform", - "ExperimentAnalysis", "Analysis", "CLIReporter", "JupyterNotebookReporter", - "ProgressReporter", "report", "get_trial_dir", "get_trial_name", - "get_trial_id", "make_checkpoint_dir", "save_checkpoint", + "lograndint", "qrandint", "qlograndint", "randn", "qrandn", "loguniform", + "qloguniform", "ExperimentAnalysis", "Analysis", "CLIReporter", + "JupyterNotebookReporter", "ProgressReporter", "report", "get_trial_dir", + "get_trial_name", "get_trial_id", "make_checkpoint_dir", "save_checkpoint", "is_session_enabled", "checkpoint_dir", "SyncConfig", "create_searcher", "create_scheduler" ] diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py index 7190c69d2..4a2180b5d 100644 --- a/python/ray/tune/sample.py +++ b/python/ray/tune/sample.py @@ -228,6 +228,22 @@ class Integer(Domain): items = np.random.randint(domain.lower, domain.upper, size=size) return items if len(items) > 1 else domain.cast(items[0]) + class _LogUniform(LogUniform): + def sample(self, + domain: "Integer", + spec: Optional[Union[List[Dict], Dict]] = None, + size: int = 1): + assert domain.lower > 0, \ + "LogUniform needs a lower bound greater than 0" + assert 0 < domain.upper < float("inf"), \ + "LogUniform needs a upper bound greater than 0" + logmin = np.log(domain.lower) / np.log(self.base) + logmax = np.log(domain.upper) / np.log(self.base) + + items = self.base**(np.random.uniform(logmin, logmax, size=size)) + items = np.round(items).astype(int) + return items if len(items) > 1 else domain.cast(items[0]) + default_sampler_cls = _Uniform def __init__(self, lower, upper): @@ -247,6 +263,23 @@ class Integer(Domain): new.set_sampler(self._Uniform()) return new + def loguniform(self, base: float = 10): + if not self.lower > 0: + raise ValueError( + "LogUniform requires a lower bound greater than 0." + f"Got: {self.lower}. Did you pass a variable that has " + "been log-transformed? If so, pass the non-transformed value " + "instead.") + if not 0 < self.upper < float("inf"): + raise ValueError( + "LogUniform requires a upper bound greater than 0. " + f"Got: {self.lower}. Did you pass a variable that has " + "been log-transformed? If so, pass the non-transformed value " + "instead.") + new = copy(self) + new.set_sampler(self._LogUniform(base)) + return new + def is_valid(self, value: int): return self.lower <= value <= self.upper @@ -445,6 +478,16 @@ def randint(lower: int, upper: int): return Integer(lower, upper).uniform() +def lograndint(lower: int, upper: int, base: float = 10): + """Sample an integer value log-uniformly between ``lower`` and ``upper``, + with ``base`` being the base of logarithm. + + ``lower`` is inclusive, ``upper`` is exclusive. + + """ + return Integer(lower, upper).loguniform(base) + + def qrandint(lower: int, upper: int, q: int = 1): """Sample an integer value uniformly between ``lower`` and ``upper``. @@ -453,13 +496,23 @@ def qrandint(lower: int, upper: int, q: int = 1): The value will be quantized, i.e. rounded to an integer increment of ``q``. Quantization makes the upper bound inclusive. - Sampling from ``tune.randint(10)`` is equivalent to sampling from - ``np.random.randint(10)`` - """ return Integer(lower, upper).uniform().quantized(q) +def qlograndint(lower: int, upper: int, q: int, base: float = 10): + """Sample an integer value log-uniformly between ``lower`` and ``upper``, + with ``base`` being the base of logarithm. + + ``lower`` is inclusive, ``upper`` is also inclusive (!). + + The value will be quantized, i.e. rounded to an integer increment of ``q``. + Quantization makes the upper bound inclusive. + + """ + return Integer(lower, upper).loguniform(base).quantized(q) + + def randn(mean: float = 0., sd: float = 1.): """Sample a float value normally with ``mean`` and ``sd``. diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index b173d5e85..21de8fe14 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -281,7 +281,15 @@ class TuneBOHB(Searcher): log=False) elif isinstance(domain, Integer): - if isinstance(sampler, Uniform): + if isinstance(sampler, LogUniform): + lower = domain.lower + upper = domain.upper + if quantize: + lower = math.ceil(domain.lower / quantize) * quantize + upper = math.floor(domain.upper / quantize) * quantize + return ConfigSpace.UniformIntegerHyperparameter( + par, lower=lower, upper=upper, q=quantize, log=True) + elif isinstance(sampler, Uniform): lower = domain.lower upper = domain.upper if quantize: diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py index aee5fd82d..a5b81f2ed 100644 --- a/python/ray/tune/suggest/hyperopt.py +++ b/python/ray/tune/suggest/hyperopt.py @@ -400,8 +400,9 @@ class HyperOptSearch(Searcher): if isinstance(domain, Float): if isinstance(sampler, LogUniform): if quantize: - return hpo.hp.qloguniform(par, domain.lower, - domain.upper, quantize) + return hpo.hp.qloguniform(par, np.log(domain.lower), + np.log(domain.upper), + quantize) return hpo.hp.loguniform(par, np.log(domain.lower), np.log(domain.upper)) elif isinstance(sampler, Uniform): @@ -416,12 +417,21 @@ class HyperOptSearch(Searcher): return hpo.hp.normal(par, sampler.mean, sampler.sd) elif isinstance(domain, Integer): - if isinstance(sampler, Uniform): + if isinstance(sampler, LogUniform): if quantize: - logger.warning( - "HyperOpt does not support quantization for " - "integer values. Reverting back to 'randint'.") - return hpo.hp.randint(par, domain.lower, high=domain.upper) + return hpo.base.pyll.scope.int( + hpo.hp.qloguniform(par, np.log(domain.lower), + np.log(domain.upper), quantize)) + return hpo.base.pyll.scope.int( + hpo.hp.qloguniform(par, np.log(domain.lower), + np.log(domain.upper), 1.0)) + elif isinstance(sampler, Uniform): + if quantize: + return hpo.base.pyll.scope.int( + hpo.hp.quniform(par, domain.lower, domain.upper, + quantize)) + return hpo.hp.uniformint( + par, domain.lower, high=domain.upper) elif isinstance(domain, Categorical): if isinstance(sampler, Uniform): return hpo.hp.choice(par, [ diff --git a/python/ray/tune/suggest/nevergrad.py b/python/ray/tune/suggest/nevergrad.py index f5da80b00..8df7269ae 100644 --- a/python/ray/tune/suggest/nevergrad.py +++ b/python/ray/tune/suggest/nevergrad.py @@ -310,16 +310,23 @@ class NevergradSearch(Searcher): exponent=sampler.base) return ng.p.Scalar(lower=domain.lower, upper=domain.upper) - if isinstance(domain, Integer): + elif isinstance(domain, Integer): + if isinstance(sampler, LogUniform): + return ng.p.Log( + lower=domain.lower, + upper=domain.upper, + exponent=sampler.base).set_integer_casting() return ng.p.Scalar( lower=domain.lower, upper=domain.upper).set_integer_casting() - if isinstance(domain, Categorical): + elif isinstance(domain, Categorical): return ng.p.Choice(choices=domain.categories) - raise ValueError("SkOpt does not support parameters of type " - "`{}`".format(type(domain).__name__)) + raise ValueError("Nevergrad does not support parameters of type " + "`{}` with samplers of type `{}`".format( + type(domain).__name__, + type(domain.sampler).__name__)) # Parameter name is e.g. "a/b/c" for nested dicts space = { diff --git a/python/ray/tune/suggest/skopt.py b/python/ray/tune/suggest/skopt.py index 574be4f35..7c4f337af 100644 --- a/python/ray/tune/suggest/skopt.py +++ b/python/ray/tune/suggest/skopt.py @@ -4,7 +4,8 @@ import pickle from typing import Dict, List, Optional, Tuple, Union from ray.tune.result import DEFAULT_METRIC -from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized +from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized, \ + LogUniform from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars @@ -334,24 +335,26 @@ class SkOptSearch(Searcher): sampler = sampler.get_sampler() if isinstance(domain, Float): - if domain.sampler is not None: - logger.warning( - "SkOpt does not support specific sampling methods." - " The {} sampler will be dropped.".format(sampler)) - return domain.lower, domain.upper + if isinstance(domain.sampler, LogUniform): + return sko.space.Real( + domain.lower, domain.upper, prior="log-uniform") + return sko.space.Real( + domain.lower, domain.upper, prior="uniform") - if isinstance(domain, Integer): - if domain.sampler is not None: - logger.warning( - "SkOpt does not support specific sampling methods." - " The {} sampler will be dropped.".format(sampler)) - return domain.lower, domain.upper + elif isinstance(domain, Integer): + if isinstance(domain.sampler, LogUniform): + return sko.space.Integer( + domain.lower, domain.upper, prior="log-uniform") + return sko.space.Integer( + domain.lower, domain.upper, prior="uniform") - if isinstance(domain, Categorical): + elif isinstance(domain, Categorical): return domain.categories raise ValueError("SkOpt does not support parameters of type " - "`{}`".format(type(domain).__name__)) + "`{}` with samplers of type `{}`".format( + type(domain).__name__, + type(domain.sampler).__name__)) # Parameter name is e.g. "a/b/c" for nested dicts space = { diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py index 8a06be5d0..4c1d1a1cb 100644 --- a/python/ray/tune/tests/test_sample.py +++ b/python/ray/tune/tests/test_sample.py @@ -26,7 +26,9 @@ class SearchSpaceTest(unittest.TestCase): "qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5), "choice": tune.choice([2, 3, 4]), "randint": tune.randint(-9, 15), + "lograndint": tune.lograndint(1, 10), "qrandint": tune.qrandint(-21, 12, 3), + "qlograndint": tune.qlograndint(2, 20, 2), "randn": tune.randn(10, 2), "qrandn": tune.qrandn(10, 2, 0.2), } @@ -58,10 +60,21 @@ class SearchSpaceTest(unittest.TestCase): self.assertGreaterEqual(out["randint"], -9) self.assertLess(out["randint"], 15) + self.assertTrue(isinstance(out["randint"], int)) + + self.assertGreaterEqual(out["lograndint"], 1) + self.assertLess(out["lograndint"], 10) + self.assertTrue(isinstance(out["lograndint"], int)) self.assertGreaterEqual(out["qrandint"], -21) self.assertLessEqual(out["qrandint"], 12) self.assertEqual(out["qrandint"] % 3, 0) + self.assertTrue(isinstance(out["qrandint"], int)) + + self.assertGreaterEqual(out["qlograndint"], 2) + self.assertLessEqual(out["qlograndint"], 20) + self.assertEqual(out["qlograndint"] % 2, 0) + self.assertTrue(isinstance(out["qlograndint"], int)) # Very improbable self.assertGreater(out["randn"], 0) @@ -417,7 +430,7 @@ class SearchSpaceTest(unittest.TestCase): config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { - "x": tune.sample.Integer(-15, -10).quantized(2), + "x": tune.sample.Integer(-15, -10), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } @@ -426,7 +439,7 @@ class SearchSpaceTest(unittest.TestCase): hyperopt_config = { "a": hp.choice("a", [2, 3, 4]), "b": { - "x": hp.randint("x", -15, -10), + "x": hp.uniformint("x", -15, -10), "y": 4, "z": hp.loguniform("z", np.log(1e-4), np.log(1e-2)) } @@ -625,17 +638,22 @@ class SearchSpaceTest(unittest.TestCase): def testConvertSkOpt(self): from ray.tune.suggest.skopt import SkOptSearch + from skopt.space import Real, Integer config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { - "x": tune.sample.Integer(0, 5).quantized(2), + "x": tune.sample.Integer(0, 5), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = SkOptSearch.convert_search_space(config) - skopt_config = {"a": [2, 3, 4], "b/x": (0, 5), "b/z": (1e-4, 1e-2)} + skopt_config = { + "a": [2, 3, 4], + "b/x": Integer(0, 5), + "b/z": Real(1e-4, 1e-2, prior="log-uniform") + } searcher1 = SkOptSearch(space=converted_config, metric="a", mode="max") searcher2 = SkOptSearch(space=skopt_config, metric="a", mode="max")