[tune] Fix and enable SigOpt tests (#12877)

Co-authored-by: Richard Liaw <rliaw@berkeley.edu>

python/ray/tune/suggest/bayesopt.py

@@ -169,9 +169,7 @@ class BayesOptSearch(Searcher):
 
         self.utility = byo.UtilityFunction(**utility_kwargs)
 
-        # Registering the provided analysis, if given
-        if analysis is not None:
-            self.register_analysis(analysis)
+        self._analysis = analysis
 
         if isinstance(space, dict) and space:
             resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
@@ -200,6 +198,10 @@ class BayesOptSearch(Searcher):
             verbose=self._verbose,
             random_state=self._random_state)
 
+        # Registering the provided analysis, if given
+        if self._analysis is not None:
+            self.register_analysis(self._analysis)
+
     def set_search_properties(self, metric: Optional[str], mode: Optional[str],
                               config: Dict) -> bool:
         if self.optimizer:

python/ray/tune/suggest/nevergrad.py

@@ -148,7 +148,7 @@ class NevergradSearch(Searcher):
                 space = self.convert_search_space(space)
 
         if isinstance(optimizer, Optimizer):
-            if space is not None or isinstance(space, list):
+            if space is not None and not isinstance(space, list):
                 raise ValueError(
                     "If you pass a configured optimizer to Nevergrad, either "
                     "pass a list of parameter names or None as the `space` "

python/ray/tune/suggest/sigopt.py

@@ -136,6 +136,7 @@ class SigOptSearch(Searcher):
                  project: Optional[str] = None,
                  metric: Union[None, str, List[str]] = "episode_reward_mean",
                  mode: Union[None, str, List[str]] = "max",
+                 points_to_evaluate: Optional[List[Dict]] = None,
                  **kwargs):
         assert (experiment_id is
                 None) ^ (space is None), "space xor experiment_id must be set"
@@ -182,17 +183,25 @@ class SigOptSearch(Searcher):
         else:
             self.experiment = self.conn.experiments(experiment_id).fetch()
 
+        self._points_to_evaluate = points_to_evaluate
+
         super(SigOptSearch, self).__init__(metric=metric, mode=mode, **kwargs)
 
     def suggest(self, trial_id: str):
         if self._max_concurrent:
             if len(self._live_trial_mapping) >= self._max_concurrent:
                 return None
+
+        suggestion_kwargs = {}
+        if self._points_to_evaluate:
+            config = self._points_to_evaluate.pop(0)
+            suggestion_kwargs = {"assignments": config}
+
         # Get new suggestion from SigOpt
         suggestion = self.conn.experiments(
-            self.experiment.id).suggestions().create()
+            self.experiment.id).suggestions().create(**suggestion_kwargs)
 
-        self._live_trial_mapping[trial_id] = suggestion
+        self._live_trial_mapping[trial_id] = suggestion.id
 
         return copy.deepcopy(suggestion.assignments)
 
@@ -210,7 +219,7 @@ class SigOptSearch(Searcher):
         """
         if result:
             payload = dict(
-                suggestion=self._live_trial_mapping[trial_id].id,
+                suggestion=self._live_trial_mapping[trial_id],
                 values=self.serialize_result(result))
             self.conn.experiments(
                 self.experiment.id).observations().create(**payload)
@@ -219,7 +228,7 @@ class SigOptSearch(Searcher):
         elif error:
             # Reports a failed Observation
             self.conn.experiments(self.experiment.id).observations().create(
-                failed=True, suggestion=self._live_trial_mapping[trial_id].id)
+                failed=True, suggestion=self._live_trial_mapping[trial_id])
         del self._live_trial_mapping[trial_id]
 
     @staticmethod
@@ -254,12 +263,15 @@ class SigOptSearch(Searcher):
         return values
 
     def save(self, checkpoint_path: str):
-        trials_object = (self.conn, self.experiment)
+        trials_object = (self.experiment.id, self._live_trial_mapping,
+                         self._points_to_evaluate)
         with open(checkpoint_path, "wb") as outputFile:
             pickle.dump(trials_object, outputFile)
 
     def restore(self, checkpoint_path: str):
         with open(checkpoint_path, "rb") as inputFile:
             trials_object = pickle.load(inputFile)
-        self.conn = trials_object[0]
-        self.experiment = trials_object[1]
+        experiment_id, self._live_trial_mapping, self._points_to_evaluate = \
+            trials_object
+
+        self.experiment = self.conn.experiments(experiment_id).fetch()

python/ray/tune/suggest/suggestion.py

@@ -391,6 +391,12 @@ class ConcurrencyLimiter(Searcher):
     def set_state(self, state: Dict):
         self.__dict__.update(state)
 
+    def save(self, checkpoint_path: str):
+        self.searcher.save(checkpoint_path)
+
+    def restore(self, checkpoint_path: str):
+        self.searcher.restore(checkpoint_path)
+
     def on_pause(self, trial_id: str):
         self.searcher.on_pause(trial_id)
 

python/ray/tune/suggest/zoopt.py

@@ -138,6 +138,7 @@ class ZOOptSearch(Searcher):
                  metric: Optional[str] = None,
                  mode: Optional[str] = None,
                  points_to_evaluate: Optional[List[Dict]] = None,
+                 parallel_num: int = 1,
                  **kwargs):
         assert zoopt is not None, "ZOOpt not found - please install zoopt " \
                                   "by `pip install -U zoopt`."
@@ -178,6 +179,8 @@ class ZOOptSearch(Searcher):
 
         self.kwargs = kwargs
 
+        self.parallel_num = parallel_num
+
         super(ZOOptSearch, self).__init__(metric=self._metric, mode=mode)
 
         if self._dim_dict:
@@ -206,7 +209,10 @@ class ZOOptSearch(Searcher):
         if self._algo == "sracos" or self._algo == "asracos":
             from zoopt.algos.opt_algorithms.racos.sracos import SRacosTune
             self.optimizer = SRacosTune(
-                dimension=dim, parameter=par, **self.kwargs)
+                dimension=dim,
+                parameter=par,
+                parallel_num=self.parallel_num,
+                **self.kwargs)
             if init_samples:
                 self.optimizer.init_attribute()