diff --git a/python/ray/rllib/ppo/filter.py b/python/ray/rllib/ppo/filter.py
index d61306b45..bd424ebd3 100644
--- a/python/ray/rllib/ppo/filter.py
+++ b/python/ray/rllib/ppo/filter.py
@@ -128,7 +128,7 @@ class MeanStdFilter(object):
             else:
                 # The unvectorized case.
                 self.rs.push(x)
-                self.buffer.push(x[i])
+                self.buffer.push(x)
         if self.demean:
             x = x - self.rs.mean
         if self.destd: