[tune] fix non-deterministic category sampling by switching back to np.random.choice (#13710)

* Enable zoopt tests again, but wait for next release

* Add test and preserve state in trial executor

* Add baseline check with integers

* [tune] fix non-deterministic category sampling, re-enable zoopt tests

* Remove random import

* Disable zoopt tests

python/ray/tune/ray_trial_executor.py

@@ -573,6 +573,7 @@ class RayTrialExecutor(TrialExecutor):
             return None
         shuffled_results = list(self._running.keys())
         random.shuffle(shuffled_results)
+
         # Note: We shuffle the results because `ray.wait` by default returns
         # the first available result, and we want to guarantee that slower
         # trials (i.e. trials that run remotely) also get fairly reported.

python/ray/tune/sample.py

@@ -1,5 +1,4 @@
 import logging
-import random
 from copy import copy
 from inspect import signature
 from math import isclose
@@ -295,7 +294,7 @@ class Categorical(Domain):
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
 
-            items = random.choices(domain.categories, k=size)
+            items = np.random.choice(domain.categories, size=size).tolist()
             return items if len(items) > 1 else domain.cast(items[0])
 
     default_sampler_cls = _Uniform
@@ -471,7 +470,7 @@ def choice(categories: List):
     """Sample a categorical value.
 
     Sampling from ``tune.choice([1, 2])`` is equivalent to sampling from
-    ``random.choice([1, 2])``
+    ``np.random.choice([1, 2])``
 
     """
     return Categorical(categories).uniform()

python/ray/tune/suggest/zoopt.py

@@ -198,8 +198,8 @@ class ZOOptSearch(Searcher):
 
         init_samples = None
         if self._points_to_evaluate:
-            logger.warning(
-                "`points_to_evaluate` seems to be ignored by ZOOpt.")
+            logger.warning("`points_to_evaluate` is ignored by ZOOpt in "
+                           "versions <= 0.4.1.")
             init_samples = [
                 Solution(x=tuple(point[dim] for dim in self._dim_keys))
                 for point in self._points_to_evaluate
@@ -213,8 +213,6 @@ class ZOOptSearch(Searcher):
                 parameter=par,
                 parallel_num=self.parallel_num,
                 **self.kwargs)
-            if init_samples:
-                self.optimizer.init_attribute()
 
     def set_search_properties(self, metric: Optional[str], mode: Optional[str],
                               config: Dict) -> bool:

python/ray/tune/tests/test_sample.py

@@ -193,6 +193,32 @@ class SearchSpaceTest(unittest.TestCase):
         samples = tune.sample.Float(0, 33).quantized(3).sample(size=1000)
         self.assertTrue(all(0 <= s <= 33 for s in samples))
 
+    def testCategoricalSeedInTrainingLoop(self):
+        def train(config):
+            return 0
+
+        config = {
+            "integer": tune.randint(0, 100_000),
+            "choice": tune.choice(list(range(100_000)))
+        }
+
+        np.random.seed(1000)
+
+        out_1 = tune.run(train, config=config, num_samples=8, verbose=0)
+
+        integers_1 = [t.config["integer"] for t in out_1.trials]
+        choices_1 = [t.config["choice"] for t in out_1.trials]
+
+        np.random.seed(1000)
+
+        out_2 = tune.run(train, config=config, num_samples=8, verbose=0)
+
+        integers_2 = [t.config["integer"] for t in out_2.trials]
+        choices_2 = [t.config["choice"] for t in out_2.trials]
+
+        self.assertSequenceEqual(integers_1, integers_2)
+        self.assertSequenceEqual(choices_1, choices_2)
+
     def testConvertAx(self):
         from ray.tune.suggest.ax import AxSearch
         from ax.service.ax_client import AxClient
@@ -952,9 +978,11 @@ class SearchSpaceTest(unittest.TestCase):
         return self._testPointsToEvaluate(SkOptSearch, config)
 
     def testPointsToEvaluateZoOpt(self):
-        # https://github.com/polixir/ZOOpt/issues/5
-        self.skipTest("ZoOpt currently ignores initial points. This test "
-                      "will be enabled after this has been fixed.")
+        self.skipTest(
+            "ZOOpt's latest release (0.4.1) does not support sampling "
+            "initial points. Please re-enable this test after the next "
+            "release.")
+
         config = {
             "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(),
             "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(),