[rllib] Add async remote workers (#4253)

python/ray/rllib/agents/agent.py

@@ -134,6 +134,9 @@ COMMON_CONFIG = {
     # remote processes instead of in the same worker. This adds overheads, but
     # can make sense if your envs are very CPU intensive (e.g., for StarCraft).
     "remote_worker_envs": False,
+    # Similar to remote_worker_envs, but runs the envs asynchronously in the
+    # background for greater efficiency. Conflicts with remote_worker_envs.
+    "async_remote_worker_envs": False,
 
     # === Offline Datasets ===
     # __sphinx_doc_input_begin__
@@ -473,9 +476,7 @@ class Agent(Trainable):
                         "tf_session_args": self.
                         config["local_evaluator_tf_session_args"]
                     }),
-                extra_config or {}),
-            remote_worker_envs=False,
-        )
+                extra_config or {}))
 
     @DeveloperAPI
     def make_remote_evaluators(self, env_creator, policy_graph, count):
@@ -490,14 +491,8 @@ class Agent(Trainable):
         cls = PolicyEvaluator.as_remote(**remote_args).remote
 
         return [
-            self._make_evaluator(
-                cls,
-                env_creator,
-                policy_graph,
-                i + 1,
-                self.config,
-                remote_worker_envs=self.config["remote_worker_envs"])
-            for i in range(count)
+            self._make_evaluator(cls, env_creator, policy_graph, i + 1,
+                                 self.config) for i in range(count)
         ]
 
     @DeveloperAPI
@@ -563,13 +558,8 @@ class Agent(Trainable):
                 "`input_evaluation` must be a list of strings, got {}".format(
                     config["input_evaluation"]))
 
-    def _make_evaluator(self,
-                        cls,
-                        env_creator,
-                        policy_graph,
-                        worker_index,
-                        config,
-                        remote_worker_envs=False):
+    def _make_evaluator(self, cls, env_creator, policy_graph, worker_index,
+                        config):
         def session_creator():
             logger.debug("Creating TF session {}".format(
                 config["tf_session_args"]))
@@ -639,7 +629,8 @@ class Agent(Trainable):
             input_creator=input_creator,
             input_evaluation=input_evaluation,
             output_creator=output_creator,
-            remote_worker_envs=remote_worker_envs)
+            remote_worker_envs=config["remote_worker_envs"],
+            async_remote_worker_envs=config["async_remote_worker_envs"])
 
     @override(Trainable)
     def _export_model(self, export_formats, export_dir):