[Tune]Add integer loguniform support (#12994)

* Add integer quantization and loguniform support

* Fix hyperopt qloguniform not being np.log'd first

* Add tests, __init__

* Try to fix tests, better exceptions

* Tweak docstrings

* Type checks in SearchSpaceTest

* Update docs

* Lint, tests

* Update doc/source/tune/api_docs/search_space.rst

Co-authored-by: Kai Fricke <krfricke@users.noreply.github.com>

Co-authored-by: Kai Fricke <krfricke@users.noreply.github.com>

python/ray/tune/__init__.py

@@ -16,8 +16,8 @@ from ray.tune.session import (
 from ray.tune.progress_reporter import (ProgressReporter, CLIReporter,
                                         JupyterNotebookReporter)
 from ray.tune.sample import (function, sample_from, uniform, quniform, choice,
-                             randint, qrandint, randn, qrandn, loguniform,
-                             qloguniform)
+                             randint, lograndint, qrandint, qlograndint, randn,
+                             qrandn, loguniform, qloguniform)
 from ray.tune.suggest import create_searcher
 from ray.tune.schedulers import create_scheduler
 
@@ -26,10 +26,10 @@ __all__ = [
     "register_env", "register_trainable", "run", "run_experiments",
     "with_parameters", "Stopper", "EarlyStopping", "Experiment", "function",
     "sample_from", "track", "uniform", "quniform", "choice", "randint",
-    "qrandint", "randn", "qrandn", "loguniform", "qloguniform",
-    "ExperimentAnalysis", "Analysis", "CLIReporter", "JupyterNotebookReporter",
-    "ProgressReporter", "report", "get_trial_dir", "get_trial_name",
-    "get_trial_id", "make_checkpoint_dir", "save_checkpoint",
+    "lograndint", "qrandint", "qlograndint", "randn", "qrandn", "loguniform",
+    "qloguniform", "ExperimentAnalysis", "Analysis", "CLIReporter",
+    "JupyterNotebookReporter", "ProgressReporter", "report", "get_trial_dir",
+    "get_trial_name", "get_trial_id", "make_checkpoint_dir", "save_checkpoint",
     "is_session_enabled", "checkpoint_dir", "SyncConfig", "create_searcher",
     "create_scheduler"
 ]

python/ray/tune/sample.py

@@ -228,6 +228,22 @@ class Integer(Domain):
             items = np.random.randint(domain.lower, domain.upper, size=size)
             return items if len(items) > 1 else domain.cast(items[0])
 
+    class _LogUniform(LogUniform):
+        def sample(self,
+                   domain: "Integer",
+                   spec: Optional[Union[List[Dict], Dict]] = None,
+                   size: int = 1):
+            assert domain.lower > 0, \
+                "LogUniform needs a lower bound greater than 0"
+            assert 0 < domain.upper < float("inf"), \
+                "LogUniform needs a upper bound greater than 0"
+            logmin = np.log(domain.lower) / np.log(self.base)
+            logmax = np.log(domain.upper) / np.log(self.base)
+
+            items = self.base**(np.random.uniform(logmin, logmax, size=size))
+            items = np.round(items).astype(int)
+            return items if len(items) > 1 else domain.cast(items[0])
+
     default_sampler_cls = _Uniform
 
     def __init__(self, lower, upper):
@@ -247,6 +263,23 @@ class Integer(Domain):
         new.set_sampler(self._Uniform())
         return new
 
+    def loguniform(self, base: float = 10):
+        if not self.lower > 0:
+            raise ValueError(
+                "LogUniform requires a lower bound greater than 0."
+                f"Got: {self.lower}. Did you pass a variable that has "
+                "been log-transformed? If so, pass the non-transformed value "
+                "instead.")
+        if not 0 < self.upper < float("inf"):
+            raise ValueError(
+                "LogUniform requires a upper bound greater than 0. "
+                f"Got: {self.lower}. Did you pass a variable that has "
+                "been log-transformed? If so, pass the non-transformed value "
+                "instead.")
+        new = copy(self)
+        new.set_sampler(self._LogUniform(base))
+        return new
+
     def is_valid(self, value: int):
         return self.lower <= value <= self.upper
 
@@ -445,6 +478,16 @@ def randint(lower: int, upper: int):
     return Integer(lower, upper).uniform()
 
 
+def lograndint(lower: int, upper: int, base: float = 10):
+    """Sample an integer value log-uniformly between ``lower`` and ``upper``,
+    with ``base`` being the base of logarithm.
+
+    ``lower`` is inclusive, ``upper`` is exclusive.
+
+    """
+    return Integer(lower, upper).loguniform(base)
+
+
 def qrandint(lower: int, upper: int, q: int = 1):
     """Sample an integer value uniformly between ``lower`` and ``upper``.
 
@@ -453,13 +496,23 @@ def qrandint(lower: int, upper: int, q: int = 1):
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
     Quantization makes the upper bound inclusive.
 
-    Sampling from ``tune.randint(10)`` is equivalent to sampling from
-    ``np.random.randint(10)``
-
     """
     return Integer(lower, upper).uniform().quantized(q)
 
 
+def qlograndint(lower: int, upper: int, q: int, base: float = 10):
+    """Sample an integer value log-uniformly between ``lower`` and ``upper``,
+    with ``base`` being the base of logarithm.
+
+    ``lower`` is inclusive, ``upper`` is also inclusive (!).
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+    Quantization makes the upper bound inclusive.
+
+    """
+    return Integer(lower, upper).loguniform(base).quantized(q)
+
+
 def randn(mean: float = 0., sd: float = 1.):
     """Sample a float value normally with ``mean`` and ``sd``.
 

python/ray/tune/suggest/bohb.py

@@ -281,7 +281,15 @@ class TuneBOHB(Searcher):
                         log=False)
 
             elif isinstance(domain, Integer):
-                if isinstance(sampler, Uniform):
+                if isinstance(sampler, LogUniform):
+                    lower = domain.lower
+                    upper = domain.upper
+                    if quantize:
+                        lower = math.ceil(domain.lower / quantize) * quantize
+                        upper = math.floor(domain.upper / quantize) * quantize
+                    return ConfigSpace.UniformIntegerHyperparameter(
+                        par, lower=lower, upper=upper, q=quantize, log=True)
+                elif isinstance(sampler, Uniform):
                     lower = domain.lower
                     upper = domain.upper
                     if quantize:

python/ray/tune/suggest/hyperopt.py

@@ -400,8 +400,9 @@ class HyperOptSearch(Searcher):
             if isinstance(domain, Float):
                 if isinstance(sampler, LogUniform):
                     if quantize:
-                        return hpo.hp.qloguniform(par, domain.lower,
-                                                  domain.upper, quantize)
+                        return hpo.hp.qloguniform(par, np.log(domain.lower),
+                                                  np.log(domain.upper),
+                                                  quantize)
                     return hpo.hp.loguniform(par, np.log(domain.lower),
                                              np.log(domain.upper))
                 elif isinstance(sampler, Uniform):
@@ -416,12 +417,21 @@ class HyperOptSearch(Searcher):
                     return hpo.hp.normal(par, sampler.mean, sampler.sd)
 
             elif isinstance(domain, Integer):
-                if isinstance(sampler, Uniform):
+                if isinstance(sampler, LogUniform):
                     if quantize:
-                        logger.warning(
-                            "HyperOpt does not support quantization for "
-                            "integer values. Reverting back to 'randint'.")
-                    return hpo.hp.randint(par, domain.lower, high=domain.upper)
+                        return hpo.base.pyll.scope.int(
+                            hpo.hp.qloguniform(par, np.log(domain.lower),
+                                               np.log(domain.upper), quantize))
+                    return hpo.base.pyll.scope.int(
+                        hpo.hp.qloguniform(par, np.log(domain.lower),
+                                           np.log(domain.upper), 1.0))
+                elif isinstance(sampler, Uniform):
+                    if quantize:
+                        return hpo.base.pyll.scope.int(
+                            hpo.hp.quniform(par, domain.lower, domain.upper,
+                                            quantize))
+                    return hpo.hp.uniformint(
+                        par, domain.lower, high=domain.upper)
             elif isinstance(domain, Categorical):
                 if isinstance(sampler, Uniform):
                     return hpo.hp.choice(par, [

python/ray/tune/suggest/nevergrad.py

@@ -310,16 +310,23 @@ class NevergradSearch(Searcher):
                         exponent=sampler.base)
                 return ng.p.Scalar(lower=domain.lower, upper=domain.upper)
 
-            if isinstance(domain, Integer):
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    return ng.p.Log(
+                        lower=domain.lower,
+                        upper=domain.upper,
+                        exponent=sampler.base).set_integer_casting()
                 return ng.p.Scalar(
                     lower=domain.lower,
                     upper=domain.upper).set_integer_casting()
 
-            if isinstance(domain, Categorical):
+            elif isinstance(domain, Categorical):
                 return ng.p.Choice(choices=domain.categories)
 
-            raise ValueError("SkOpt does not support parameters of type "
-                             "`{}`".format(type(domain).__name__))
+            raise ValueError("Nevergrad does not support parameters of type "
+                             "`{}` with samplers of type `{}`".format(
+                                 type(domain).__name__,
+                                 type(domain.sampler).__name__))
 
         # Parameter name is e.g. "a/b/c" for nested dicts
         space = {

python/ray/tune/suggest/skopt.py

@@ -4,7 +4,8 @@ import pickle
 from typing import Dict, List, Optional, Tuple, Union
 
 from ray.tune.result import DEFAULT_METRIC
-from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized
+from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized, \
+    LogUniform
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
@@ -334,24 +335,26 @@ class SkOptSearch(Searcher):
                 sampler = sampler.get_sampler()
 
             if isinstance(domain, Float):
-                if domain.sampler is not None:
-                    logger.warning(
-                        "SkOpt does not support specific sampling methods."
-                        " The {} sampler will be dropped.".format(sampler))
-                return domain.lower, domain.upper
+                if isinstance(domain.sampler, LogUniform):
+                    return sko.space.Real(
+                        domain.lower, domain.upper, prior="log-uniform")
+                return sko.space.Real(
+                    domain.lower, domain.upper, prior="uniform")
 
-            if isinstance(domain, Integer):
-                if domain.sampler is not None:
-                    logger.warning(
-                        "SkOpt does not support specific sampling methods."
-                        " The {} sampler will be dropped.".format(sampler))
-                return domain.lower, domain.upper
+            elif isinstance(domain, Integer):
+                if isinstance(domain.sampler, LogUniform):
+                    return sko.space.Integer(
+                        domain.lower, domain.upper, prior="log-uniform")
+                return sko.space.Integer(
+                    domain.lower, domain.upper, prior="uniform")
 
-            if isinstance(domain, Categorical):
+            elif isinstance(domain, Categorical):
                 return domain.categories
 
             raise ValueError("SkOpt does not support parameters of type "
-                             "`{}`".format(type(domain).__name__))
+                             "`{}` with samplers of type `{}`".format(
+                                 type(domain).__name__,
+                                 type(domain.sampler).__name__))
 
         # Parameter name is e.g. "a/b/c" for nested dicts
         space = {

python/ray/tune/tests/test_sample.py

@@ -26,7 +26,9 @@ class SearchSpaceTest(unittest.TestCase):
             "qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5),
             "choice": tune.choice([2, 3, 4]),
             "randint": tune.randint(-9, 15),
+            "lograndint": tune.lograndint(1, 10),
             "qrandint": tune.qrandint(-21, 12, 3),
+            "qlograndint": tune.qlograndint(2, 20, 2),
             "randn": tune.randn(10, 2),
             "qrandn": tune.qrandn(10, 2, 0.2),
         }
@@ -58,10 +60,21 @@ class SearchSpaceTest(unittest.TestCase):
 
             self.assertGreaterEqual(out["randint"], -9)
             self.assertLess(out["randint"], 15)
+            self.assertTrue(isinstance(out["randint"], int))
+
+            self.assertGreaterEqual(out["lograndint"], 1)
+            self.assertLess(out["lograndint"], 10)
+            self.assertTrue(isinstance(out["lograndint"], int))
 
             self.assertGreaterEqual(out["qrandint"], -21)
             self.assertLessEqual(out["qrandint"], 12)
             self.assertEqual(out["qrandint"] % 3, 0)
+            self.assertTrue(isinstance(out["qrandint"], int))
+
+            self.assertGreaterEqual(out["qlograndint"], 2)
+            self.assertLessEqual(out["qlograndint"], 20)
+            self.assertEqual(out["qlograndint"] % 2, 0)
+            self.assertTrue(isinstance(out["qlograndint"], int))
 
             # Very improbable
             self.assertGreater(out["randn"], 0)
@@ -417,7 +430,7 @@ class SearchSpaceTest(unittest.TestCase):
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
-                "x": tune.sample.Integer(-15, -10).quantized(2),
+                "x": tune.sample.Integer(-15, -10),
                 "y": 4,
                 "z": tune.sample.Float(1e-4, 1e-2).loguniform()
             }
@@ -426,7 +439,7 @@ class SearchSpaceTest(unittest.TestCase):
         hyperopt_config = {
             "a": hp.choice("a", [2, 3, 4]),
             "b": {
-                "x": hp.randint("x", -15, -10),
+                "x": hp.uniformint("x", -15, -10),
                 "y": 4,
                 "z": hp.loguniform("z", np.log(1e-4), np.log(1e-2))
             }
@@ -625,17 +638,22 @@ class SearchSpaceTest(unittest.TestCase):
 
     def testConvertSkOpt(self):
         from ray.tune.suggest.skopt import SkOptSearch
+        from skopt.space import Real, Integer
 
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
-                "x": tune.sample.Integer(0, 5).quantized(2),
+                "x": tune.sample.Integer(0, 5),
                 "y": 4,
                 "z": tune.sample.Float(1e-4, 1e-2).loguniform()
             }
         }
         converted_config = SkOptSearch.convert_search_space(config)
-        skopt_config = {"a": [2, 3, 4], "b/x": (0, 5), "b/z": (1e-4, 1e-2)}
+        skopt_config = {
+            "a": [2, 3, 4],
+            "b/x": Integer(0, 5),
+            "b/z": Real(1e-4, 1e-2, prior="log-uniform")
+        }
 
         searcher1 = SkOptSearch(space=converted_config, metric="a", mode="max")
         searcher2 = SkOptSearch(space=skopt_config, metric="a", mode="max")