[tune] Deprecate ambiguous function values (use tune.function / tune.sample_from instead) (#3457)

* wip

* exclude

doc/source/tune-package-ref.rst

@@ -29,7 +29,7 @@ ray.tune.suggest
 
 .. automodule:: ray.tune.suggest
     :members:
-    :exclude-members: function, grid_search, SuggestionAlgorithm
+    :exclude-members: function, sample_from, grid_search, SuggestionAlgorithm
     :show-inheritance:
 
 .. autoclass:: ray.tune.suggest.SuggestionAlgorithm

doc/source/tune-usage.rst

@@ -141,8 +141,8 @@ The following shows grid search over two nested parameters combined with random
         "my_experiment_name": {
             "run": my_trainable,
             "config": {
-                "alpha": lambda spec: np.random.uniform(100),
-                "beta": lambda spec: spec.config.alpha * np.random.normal(),
+                "alpha": tune.sample_from(lambda spec: np.random.uniform(100)),
+                "beta": tune.sample_from(lambda spec: spec.config.alpha * np.random.normal()),
                 "nn_layers": [
                     tune.grid_search([16, 64, 256]),
                     tune.grid_search([16, 64, 256]),
@@ -153,7 +153,7 @@ The following shows grid search over two nested parameters combined with random
 
 
 .. note::
-    Lambda functions will be evaluated during trial variant generation. If you need to pass a literal function in your config, use ``tune.function(...)`` to escape it.
+    Use ``tune.sample_from(...)`` to sample from a function during trial variant generation. If you need to pass a literal function in your config, use ``tune.function(...)`` to escape it.
 
 For more information on variant generation, see `basic_variant.py <https://github.com/ray-project/ray/blob/master/python/ray/tune/suggest/basic_variant.py>`__.
 
@@ -169,8 +169,8 @@ By default, each random variable and grid search point is sampled once. To take
         "my_experiment_name": {
             "run": my_trainable,
             "config": {
-                "alpha": lambda spec: np.random.uniform(100),
-                "beta": lambda spec: spec.config.alpha * np.random.normal(),
+                "alpha": tune.sample_from(lambda spec: np.random.uniform(100)),
+                "beta": tune.sample_from(lambda spec: spec.config.alpha * np.random.normal()),
                 "nn_layers": [
                     tune.grid_search([16, 64, 256]),
                     tune.grid_search([16, 64, 256]),

python/ray/tune/__init__.py

@@ -7,9 +7,16 @@ from ray.tune.tune import run_experiments
 from ray.tune.experiment import Experiment
 from ray.tune.registry import register_env, register_trainable
 from ray.tune.trainable import Trainable
-from ray.tune.suggest import grid_search, function
+from ray.tune.suggest import grid_search, function, sample_from
 
 __all__ = [
-    "Trainable", "TuneError", "grid_search", "register_env",
-    "register_trainable", "run_experiments", "Experiment", "function"
+    "Trainable",
+    "TuneError",
+    "grid_search",
+    "register_env",
+    "register_trainable",
+    "run_experiments",
+    "Experiment",
+    "function",
+    "sample_from",
 ]

python/ray/tune/examples/async_hyperband_example.py

@@ -12,7 +12,7 @@ import random
 import numpy as np
 
 import ray
-from ray.tune import Trainable, run_experiments
+from ray.tune import Trainable, run_experiments, sample_from
 from ray.tune.schedulers import AsyncHyperBandScheduler
 
 
@@ -76,8 +76,10 @@ if __name__ == "__main__":
                     "gpu": 0
                 },
                 "config": {
-                    "width": lambda spec: 10 + int(90 * random.random()),
-                    "height": lambda spec: int(100 * random.random()),
+                    "width": sample_from(
+                        lambda spec: 10 + int(90 * random.random())),
+                    "height": sample_from(
+                        lambda spec: int(100 * random.random())),
                 },
             }
         },

python/ray/tune/examples/hyperband_example.py

@@ -12,7 +12,7 @@ import random
 import numpy as np
 
 import ray
-from ray.tune import Trainable, run_experiments, Experiment
+from ray.tune import Trainable, run_experiments, Experiment, sample_from
 from ray.tune.schedulers import HyperBandScheduler
 
 
@@ -67,8 +67,8 @@ if __name__ == "__main__":
         num_samples=20,
         stop={"training_iteration": 1 if args.smoke_test else 99999},
         config={
-            "width": lambda spec: 10 + int(90 * random.random()),
-            "height": lambda spec: int(100 * random.random())
+            "width": sample_from(lambda spec: 10 + int(90 * random.random())),
+            "height": sample_from(lambda spec: int(100 * random.random()))
         })
 
     run_experiments(exp, scheduler=hyperband)

python/ray/tune/examples/mnist_pytorch.py

@@ -182,8 +182,10 @@ if __name__ == '__main__':
                 "run": "train_mnist",
                 "num_samples": 1 if args.smoke_test else 10,
                 "config": {
-                    "lr": lambda spec: np.random.uniform(0.001, 0.1),
-                    "momentum": lambda spec: np.random.uniform(0.1, 0.9),
+                    "lr": tune.sample_from(
+                        lambda spec: np.random.uniform(0.001, 0.1)),
+                    "momentum": tune.sample_from(
+                        lambda spec: np.random.uniform(0.1, 0.9)),
                 }
             }
         },

python/ray/tune/examples/mnist_pytorch_trainable.py

@@ -195,8 +195,10 @@ if __name__ == '__main__':
                 "checkpoint_at_end": True,
                 "config": {
                     "args": args,
-                    "lr": lambda spec: np.random.uniform(0.001, 0.1),
-                    "momentum": lambda spec: np.random.uniform(0.1, 0.9),
+                    "lr": tune.sample_from(
+                        lambda spec: np.random.uniform(0.001, 0.1)),
+                    "momentum": tune.sample_from(
+                        lambda spec: np.random.uniform(0.1, 0.9)),
                 }
             }
         },

python/ray/tune/examples/pbt_ppo_example.py

@@ -13,7 +13,7 @@ from __future__ import print_function
 import random
 
 import ray
-from ray.tune import run_experiments
+from ray.tune import run_experiments, sample_from
 from ray.tune.schedulers import PopulationBasedTraining
 
 if __name__ == "__main__":
@@ -63,12 +63,12 @@ if __name__ == "__main__":
                     "clip_param": 0.2,
                     "lr": 1e-4,
                     # These params start off randomly drawn from a set.
-                    "num_sgd_iter":
-                        lambda spec: random.choice([10, 20, 30]),
-                    "sgd_minibatch_size":
-                        lambda spec: random.choice([128, 512, 2048]),
-                    "train_batch_size":
-                        lambda spec: random.choice([10000, 20000, 40000])
+                    "num_sgd_iter": sample_from(
+                        lambda spec: random.choice([10, 20, 30])),
+                    "sgd_minibatch_size": sample_from(
+                        lambda spec: random.choice([128, 512, 2048])),
+                    "train_batch_size": sample_from(
+                        lambda spec: random.choice([10000, 20000, 40000]))
                 },
             },
         },

python/ray/tune/examples/pbt_tune_cifar10_with_keras.py

@@ -23,7 +23,7 @@ from tensorflow.python.keras.models import Model
 from tensorflow.python.keras.preprocessing.image import ImageDataGenerator
 
 import ray
-from ray.tune import grid_search, run_experiments
+from ray.tune import grid_search, run_experiments, sample_from
 from ray.tune import Trainable
 from ray.tune.schedulers import PopulationBasedTraining
 
@@ -193,7 +193,7 @@ if __name__ == "__main__":
             "epochs": 1,
             "batch_size": 64,
             "lr": grid_search([10**-4, 10**-5]),
-            "decay": lambda spec: spec.config.lr / 100.0,
+            "decay": sample_from(lambda spec: spec.config.lr / 100.0),
             "dropout": grid_search([0.25, 0.5]),
         },
         "num_samples": 4,

python/ray/tune/examples/tune_mnist_keras.py

@@ -192,10 +192,14 @@ if __name__ == '__main__':
                     "gpu": 0.5 if args.use_gpu else 0
                 },
                 "config": {
-                    "lr": lambda spec: np.random.uniform(0.001, 0.1),
-                    "momentum": lambda spec: np.random.uniform(0.1, 0.9),
-                    "hidden": lambda spec: np.random.randint(32, 512),
-                    "dropout1": lambda spec: np.random.uniform(0.2, 0.8),
+                    "lr": tune.sample_from(
+                        lambda spec: np.random.uniform(0.001, 0.1)),
+                    "momentum": tune.sample_from(
+                        lambda spec: np.random.uniform(0.1, 0.9)),
+                    "hidden": tune.sample_from(
+                        lambda spec: np.random.randint(32, 512)),
+                    "dropout1": tune.sample_from(
+                        lambda spec: np.random.uniform(0.2, 0.8)),
                 }
             }
         },

python/ray/tune/examples/tune_mnist_ray_hyperband.py

@@ -31,7 +31,7 @@ import time
 
 import ray
 from ray.tune import grid_search, run_experiments, register_trainable, \
-    Trainable
+    Trainable, sample_from
 from ray.tune.schedulers import HyperBandScheduler
 from tensorflow.examples.tutorials.mnist import input_data
 
@@ -221,7 +221,8 @@ if __name__ == '__main__':
             'time_total_s': 600,
         },
         'config': {
-            'learning_rate': lambda spec: 10**np.random.uniform(-5, -3),
+            'learning_rate': sample_from(
+                lambda spec: 10**np.random.uniform(-5, -3)),
             'activation': grid_search(['relu', 'elu', 'tanh']),
         },
         "num_samples": 10,

python/ray/tune/suggest/__init__.py

@@ -2,9 +2,15 @@ from ray.tune.suggest.search import SearchAlgorithm
 from ray.tune.suggest.basic_variant import BasicVariantGenerator
 from ray.tune.suggest.suggestion import SuggestionAlgorithm
 from ray.tune.suggest.hyperopt import HyperOptSearch
-from ray.tune.suggest.variant_generator import grid_search, function
+from ray.tune.suggest.variant_generator import grid_search, function, \
+    sample_from
 
 __all__ = [
-    "SearchAlgorithm", "BasicVariantGenerator", "HyperOptSearch",
-    "SuggestionAlgorithm", "grid_search", "function"
+    "SearchAlgorithm",
+    "BasicVariantGenerator",
+    "HyperOptSearch",
+    "SuggestionAlgorithm",
+    "grid_search",
+    "function",
+    "sample_from",
 ]

python/ray/tune/suggest/variant_generator.py

@@ -3,12 +3,15 @@ from __future__ import division
 from __future__ import print_function
 
 import copy
+import logging
 import numpy
 import random
 import types
 
 from ray.tune import TuneError
 
+logger = logging.getLogger(__name__)
+
 
 def generate_variants(unresolved_spec):
     """Generates variants from a spec (dict) with unresolved values.
@@ -55,8 +58,29 @@ def grid_search(values):
     return {"grid_search": values}
 
 
+class sample_from(object):
+    """Specify that tune should sample configuration values from this function.
+
+    The use of function arguments in tune configs must be disambiguated by
+    either wrapped the function in tune.eval() or tune.function().
+
+    Arguments:
+        func: An callable function to draw a sample from.
+    """
+
+    def __init__(self, func):
+        self.func = func
+
+
 class function(object):
-    """Wraps `func` to make sure it is not expanded during resolution."""
+    """Wraps `func` to make sure it is not expanded during resolution.
+
+    The use of function arguments in tune configs must be disambiguated by
+    either wrapped the function in tune.eval() or tune.function().
+
+    Arguments:
+        func: A function literal.
+    """
 
     def __init__(self, func):
         self.func = func
@@ -203,8 +227,17 @@ def _is_resolved(v):
 
 def _try_resolve(v):
     if isinstance(v, types.FunctionType):
-        # Lambda function
+        logger.warn(
+            "Deprecation warning: Function values are ambiguous in Tune "
+            "configuations. Either wrap the function with "
+            "`tune.function(func)` to specify a function literal, or "
+            "`tune.sample_from(func)` to tell Tune to "
+            "sample values from the function during variant generation: "
+            "{}".format(v))
         return False, v
+    elif isinstance(v, sample_from):
+        # Function to sample from
+        return False, v.func
     elif isinstance(v, dict) and len(v) == 1 and "eval" in v:
         # Lambda function in eval syntax
         return False, lambda spec: eval(