[rllib] Don't shuffle samples in PPO when using lstm

python/ray/rllib/examples/cartpole_lstm.py

@@ -166,11 +166,12 @@ if __name__ == "__main__":
     tune.run_experiments({
         "test": {
             "env": "cartpole_stateless",
-            "run": "PG",
+            "run": "PPO",
             "stop": {
                 "episode_reward_mean": args.stop
             },
             "config": {
+                "num_sgd_iter": 5,
                 "model": {
                     "use_lstm": True,
                 },

python/ray/rllib/optimizers/multi_gpu_optimizer.py

@@ -108,7 +108,10 @@ class LocalMultiGPUOptimizer(PolicyOptimizer):
             value = samples[field]
             standardized = (value - value.mean()) / max(1e-4, value.std())
             samples[field] = standardized
-        samples.shuffle()
+
+        # Important: don't shuffle RNN sequence elements
+        if not self.policy._state_inputs:
+            samples.shuffle()
 
         with self.load_timer:
             tuples = self.policy._get_loss_inputs_dict(samples)