remove auto-concat of rollouts in AsyncSampler (#3556)

* remove auto-concat of rollouts in AsyncSampler

* remote auto-concat test

* remove unused reference

python/ray/rllib/evaluation/sampler.py

@@ -10,8 +10,7 @@ import six.moves.queue as queue
 import threading
 
 from ray.rllib.evaluation.episode import MultiAgentEpisode, _flatten_action
-from ray.rllib.evaluation.sample_batch import MultiAgentSampleBatchBuilder, \
-    MultiAgentBatch
+from ray.rllib.evaluation.sample_batch import MultiAgentSampleBatchBuilder
 from ray.rllib.evaluation.tf_policy_graph import TFPolicyGraph
 from ray.rllib.env.async_vector_env import AsyncVectorEnv
 from ray.rllib.env.atari_wrappers import get_wrapper_by_cls, MonitorEnv
@@ -174,20 +173,6 @@ class AsyncSampler(threading.Thread):
         if isinstance(rollout, BaseException):
             raise rollout
 
-        # We can't auto-concat rollouts in these modes
-        if self.async_vector_env.num_envs > 1 or \
-                isinstance(rollout, MultiAgentBatch):
-            return rollout
-
-        # Auto-concat rollouts; TODO(ekl) is this important for A3C perf?
-        while not rollout["dones"][-1]:
-            try:
-                part = self.queue.get_nowait()
-                if isinstance(part, BaseException):
-                    raise rollout
-                rollout = rollout.concat(part)
-            except queue.Empty:
-                break
         return rollout
 
     def get_metrics(self):

python/ray/rllib/test/test_policy_evaluator.py

@@ -264,18 +264,6 @@ class TestPolicyEvaluator(unittest.TestCase):
             self.assertIn(key, batch)
         self.assertGreater(batch["advantages"][0], 1)
 
-    def testAutoConcat(self):
-        ev = PolicyEvaluator(
-            env_creator=lambda _: MockEnv(episode_length=40),
-            policy_graph=MockPolicyGraph,
-            sample_async=True,
-            batch_steps=10,
-            batch_mode="truncate_episodes",
-            observation_filter="ConcurrentMeanStdFilter")
-        time.sleep(2)
-        batch = ev.sample()
-        self.assertEqual(batch.count, 40)  # auto-concat up to 5 episodes
-
     def testAutoVectorization(self):
         ev = PolicyEvaluator(
             env_creator=lambda cfg: MockEnv(episode_length=20, config=cfg),