[rllib] Remove need to pass around registry (#2250)

* remove registry

* fix

* too many _

* fix

* cloudpickle

* Update registry.py

* yapf

* fix test

* fix kv check

python/ray/rllib/utils/common_policy_evaluator.py

@@ -18,7 +18,6 @@ from ray.rllib.utils.sampler import AsyncSampler, SyncSampler
 from ray.rllib.utils.serving_env import ServingEnv, _ServingEnvToAsync
 from ray.rllib.utils.tf_policy_graph import TFPolicyGraph
 from ray.rllib.utils.vector_env import VectorEnv
-from ray.tune.registry import get_registry
 from ray.tune.result import TrainingResult
 
 
@@ -97,7 +96,6 @@ class CommonPolicyEvaluator(PolicyEvaluator):
             compress_observations=False,
             num_envs=1,
             observation_filter="NoFilter",
-            registry=None,
             env_config=None,
             model_config=None,
             policy_config=None):
@@ -137,15 +135,11 @@ class CommonPolicyEvaluator(PolicyEvaluator):
                 and vectorize the computation of actions. This has no effect if
                 if the env already implements VectorEnv.
             observation_filter (str): Name of observation filter to use.
-            registry (tune.Registry): User-registered objects. Pass in the
-                value from tune.registry.get_registry() if you're having
-                trouble resolving things like custom envs.
             env_config (dict): Config to pass to the env creator.
             model_config (dict): Config to use when creating the policy model.
             policy_config (dict): Config to pass to the policy.
         """
 
-        registry = registry or get_registry()
         env_config = env_config or {}
         policy_config = policy_config or {}
         model_config = model_config or {}
@@ -169,7 +163,7 @@ class CommonPolicyEvaluator(PolicyEvaluator):
         else:
             def wrap(env):
                 return ModelCatalog.get_preprocessor_as_wrapper(
-                    registry, env, model_config)
+                    env, model_config)
         self.env = wrap(self.env)
 
         def make_env():
@@ -187,11 +181,11 @@ class CommonPolicyEvaluator(PolicyEvaluator):
                 with self.sess.as_default():
                     policy = policy_graph(
                         self.env.observation_space, self.env.action_space,
-                        registry, policy_config)
+                        policy_config)
         else:
             policy = policy_graph(
                 self.env.observation_space, self.env.action_space,
-                registry, policy_config)
+                policy_config)
         self.policy_map = {
             "default": policy
         }