diff --git a/python/ray/tune/registry.py b/python/ray/tune/registry.py
index 9cda439cb..45fd2a6b9 100644
--- a/python/ray/tune/registry.py
+++ b/python/ray/tune/registry.py
@@ -3,9 +3,9 @@ from types import FunctionType
 
 import ray
 import ray.cloudpickle as pickle
-
 from ray.experimental.internal_kv import _internal_kv_initialized, \
     _internal_kv_get, _internal_kv_put
+from ray.tune.error import TuneError
 
 TRAINABLE_CLASS = "trainable_class"
 ENV_CREATOR = "env_creator"
@@ -31,9 +31,9 @@ def get_trainable_cls(trainable_name):
 
 def validate_trainable(trainable_name):
     if not has_trainable(trainable_name):
-        # Make sure rllib agents are registered
-        from ray import rllib  # noqa: F401
-        from ray.tune.error import TuneError
+        # Make sure everything rllib-related is registered.
+        from ray.rllib import _register_all
+        _register_all()
         if not has_trainable(trainable_name):
             raise TuneError("Unknown trainable: " + trainable_name)
 
diff --git a/rllib/tuned_examples/regression_tests/cartpole-es-tf.yaml b/rllib/tuned_examples/regression_tests/cartpole-es-tf.yaml
index 759ec7bcb..62e827e75 100644
--- a/rllib/tuned_examples/regression_tests/cartpole-es-tf.yaml
+++ b/rllib/tuned_examples/regression_tests/cartpole-es-tf.yaml
@@ -3,7 +3,7 @@ cartpole-es-tf:
     run: ES
     stop:
         episode_reward_mean: 150
-        timesteps_total: 400000
+        timesteps_total: 500000
     config:
         use_pytorch: false
         num_workers: 2
diff --git a/rllib/tuned_examples/regression_tests/cartpole-es-torch.yaml b/rllib/tuned_examples/regression_tests/cartpole-es-torch.yaml
index c6cb58e17..147d0638a 100644
--- a/rllib/tuned_examples/regression_tests/cartpole-es-torch.yaml
+++ b/rllib/tuned_examples/regression_tests/cartpole-es-torch.yaml
@@ -3,7 +3,7 @@ cartpole-es-torch:
     run: ES
     stop:
         episode_reward_mean: 150
-        timesteps_total: 400000
+        timesteps_total: 500000
     config:
         use_pytorch: true
         num_workers: 2