[tune] Search Alg delay import + CLI timing test (#4230)

python/ray/tune/examples/bayesopt_example.py

@@ -9,7 +9,7 @@ from __future__ import print_function
 import ray
 from ray.tune import run
 from ray.tune.schedulers import AsyncHyperBandScheduler
-from ray.tune.suggest import BayesOptSearch
+from ray.tune.suggest.bayesopt import BayesOptSearch
 
 
 def easy_objective(config, reporter):

python/ray/tune/examples/hyperopt_example.py

@@ -9,7 +9,7 @@ from __future__ import print_function
 import ray
 from ray.tune import run
 from ray.tune.schedulers import AsyncHyperBandScheduler
-from ray.tune.suggest import HyperOptSearch
+from ray.tune.suggest.hyperopt import HyperOptSearch
 
 
 def easy_objective(config, reporter):

python/ray/tune/examples/nevergrad_example.py

@@ -9,7 +9,7 @@ from __future__ import print_function
 import ray
 from ray.tune import run
 from ray.tune.schedulers import AsyncHyperBandScheduler
-from ray.tune.suggest import NevergradSearch
+from ray.tune.suggest.nevergrad import NevergradSearch
 
 
 def easy_objective(config, reporter):

python/ray/tune/examples/sigopt_example.py

@@ -9,7 +9,7 @@ from __future__ import print_function
 import ray
 from ray.tune import run
 from ray.tune.schedulers import AsyncHyperBandScheduler
-from ray.tune.suggest import SigOptSearch
+from ray.tune.suggest.sigopt import SigOptSearch
 
 
 def easy_objective(config, reporter):

python/ray/tune/examples/skopt_example.py

@@ -9,7 +9,7 @@ from __future__ import print_function
 import ray
 from ray.tune import run
 from ray.tune.schedulers import AsyncHyperBandScheduler
-from ray.tune.suggest import SkOptSearch
+from ray.tune.suggest.skopt import SkOptSearch
 
 
 def easy_objective(config, reporter):

python/ray/tune/suggest/__init__.py

@@ -1,24 +1,39 @@
 from ray.tune.suggest.search import SearchAlgorithm
 from ray.tune.suggest.basic_variant import BasicVariantGenerator
 from ray.tune.suggest.suggestion import SuggestionAlgorithm
-from ray.tune.suggest.bayesopt import BayesOptSearch
-from ray.tune.suggest.hyperopt import HyperOptSearch
-from ray.tune.suggest.nevergrad import NevergradSearch
-from ray.tune.suggest.skopt import SkOptSearch
-from ray.tune.suggest.sigopt import SigOptSearch
 from ray.tune.suggest.variant_generator import grid_search, function, \
     sample_from
 
 __all__ = [
     "SearchAlgorithm",
     "BasicVariantGenerator",
-    "BayesOptSearch",
-    "HyperOptSearch",
-    "NevergradSearch",
-    "SkOptSearch",
-    "SigOptSearch",
     "SuggestionAlgorithm",
     "grid_search",
     "function",
     "sample_from",
 ]
+
+
+def BayesOptSearch(*args, **kwargs):
+    raise DeprecationWarning("""This class has been moved. Please import via
+        `from ray.tune.suggest.bayesopt import BayesOptSearch`""")
+
+
+def HyperOptSearch(*args, **kwargs):
+    raise DeprecationWarning("""This class has been moved. Please import via
+        `from ray.tune.suggest.hyperopt import HyperOptSearch`""")
+
+
+def NevergradSearch(*args, **kwargs):
+    raise DeprecationWarning("""This class has been moved. Please import via
+        `from ray.tune.suggest.nevergrad import NevergradSearch`""")
+
+
+def SkOptSearch(*args, **kwargs):
+    raise DeprecationWarning("""This class has been moved. Please import via
+        `from ray.tune.suggest.skopt import SkOptSearch`""")
+
+
+def SigOptSearch(*args, **kwargs):
+    raise DeprecationWarning("""This class has been moved. Please import via
+        `from ray.tune.suggest.sigopt import SigOptSearch`""")

python/ray/tune/suggest/bayesopt.py

@@ -3,17 +3,13 @@ from __future__ import division
 from __future__ import print_function
 
 import copy
+try:  # Python 3 only -- needed for lint test.
+    import bayes_opt as byo
+except ImportError:
+    byo = None
 
 from ray.tune.suggest.suggestion import SuggestionAlgorithm
 
-byo = None
-
-
-def _import_bayesopt():
-    global byo
-    import bayes_opt
-    byo = bayes_opt
-
 
 class BayesOptSearch(SuggestionAlgorithm):
     """A wrapper around BayesOpt to provide trial suggestions.
@@ -50,7 +46,6 @@ class BayesOptSearch(SuggestionAlgorithm):
                  random_state=1,
                  verbose=0,
                  **kwargs):
-        _import_bayesopt()
         assert byo is not None, (
             "BayesOpt must be installed!. You can install BayesOpt with"
             " the command: `pip install bayesian-optimization`.")

python/ray/tune/suggest/hyperopt.py

@@ -5,20 +5,16 @@ from __future__ import print_function
 import numpy as np
 import copy
 import logging
+try:
+    hyperopt_logger = logging.getLogger("hyperopt")
+    hyperopt_logger.setLevel(logging.WARNING)
+    import hyperopt as hpo
+except ImportError:
+    hpo = None
 
 from ray.tune.error import TuneError
 from ray.tune.suggest.suggestion import SuggestionAlgorithm
 
-hpo = None
-
-
-def _import_hyperopt():
-    global hpo
-    hyperopt_logger = logging.getLogger("hyperopt")
-    hyperopt_logger.setLevel(logging.WARNING)
-    import hyperopt
-    hpo = hyperopt
-
 
 class HyperOptSearch(SuggestionAlgorithm):
     """A wrapper around HyperOpt to provide trial suggestions.
@@ -66,7 +62,6 @@ class HyperOptSearch(SuggestionAlgorithm):
                  reward_attr="episode_reward_mean",
                  points_to_evaluate=None,
                  **kwargs):
-        _import_hyperopt()
         assert hpo is not None, "HyperOpt must be installed!"
         from hyperopt.fmin import generate_trials_to_calculate
         assert type(max_concurrent) is int and max_concurrent > 0

python/ray/tune/suggest/nevergrad.py

@@ -3,9 +3,9 @@ from __future__ import division
 from __future__ import print_function
 
 try:
-    import nevergrad
-except Exception:
-    nevergrad = None
+    import nevergrad as ng
+except ImportError:
+    ng = None
 
 from ray.tune.suggest.suggestion import SuggestionAlgorithm
 
@@ -42,7 +42,7 @@ class NevergradSearch(SuggestionAlgorithm):
                  max_concurrent=10,
                  reward_attr="episode_reward_mean",
                  **kwargs):
-        assert nevergrad is not None, "Nevergrad must be installed!"
+        assert ng is not None, "Nevergrad must be installed!"
         assert type(max_concurrent) is int and max_concurrent > 0
         self._max_concurrent = max_concurrent
         self._parameters = parameter_names

python/ray/tune/suggest/sigopt.py

@@ -4,10 +4,9 @@ from __future__ import print_function
 
 import copy
 import os
-
 try:
     import sigopt as sgo
-except Exception:
+except ImportError:
     sgo = None
 
 from ray.tune.suggest.suggestion import SuggestionAlgorithm

python/ray/tune/suggest/skopt.py

@@ -3,9 +3,9 @@ from __future__ import division
 from __future__ import print_function
 
 try:
-    import skopt
-except Exception:
-    skopt = None
+    import skopt as sko
+except ImportError:
+    sko = None
 
 from ray.tune.suggest.suggestion import SuggestionAlgorithm
 
@@ -85,7 +85,7 @@ class SkOptSearch(SuggestionAlgorithm):
                  points_to_evaluate=None,
                  evaluated_rewards=None,
                  **kwargs):
-        assert skopt is not None, """skopt must be installed!
+        assert sko is not None, """skopt must be installed!
             You can install Skopt with the command:
             `pip install scikit-optimize`."""
         assert type(max_concurrent) is int and max_concurrent > 0

python/ray/tune/tests/test_commands.py

@@ -4,7 +4,9 @@ from __future__ import print_function
 
 import os
 import pytest
+import subprocess
 import sys
+import time
 try:
     from cStringIO import StringIO
 except ImportError:
@@ -37,6 +39,29 @@ def start_ray():
     ray.shutdown()
 
 
+def test_time(start_ray, tmpdir):
+    experiment_name = "test_time"
+    experiment_path = os.path.join(str(tmpdir), experiment_name)
+    num_samples = 2
+    tune.run_experiments({
+        experiment_name: {
+            "run": "__fake",
+            "stop": {
+                "training_iteration": 1
+            },
+            "num_samples": num_samples,
+            "local_dir": str(tmpdir)
+        }
+    })
+    times = []
+    for i in range(5):
+        start = time.time()
+        subprocess.check_call(["tune", "ls", experiment_path])
+        times += [time.time() - start]
+
+    assert sum(times) / len(times) < 2.0, "CLI is taking too long!"
+
+
 def test_ls(start_ray, tmpdir):
     """This test captures output of list_trials."""
     experiment_name = "test_ls"