diff --git a/rllib/utils/window_stat.py b/rllib/utils/window_stat.py
index 4c3221070..9aa0d9f30 100644
--- a/rllib/utils/window_stat.py
+++ b/rllib/utils/window_stat.py
@@ -16,13 +16,13 @@ class WindowStat:
 
     def stats(self):
         if not self.count:
-            quantiles = []
+            _quantiles = []
         else:
-            quantiles = np.percentile(self.items[:self.count],
-                                      [0, 10, 50, 90, 100]).tolist()
+            _quantiles = np.nanpercentile(self.items[:self.count],
+                                          [0, 10, 50, 90, 100]).tolist()
         return {
             self.name + "_count": int(self.count),
-            self.name + "_mean": float(np.mean(self.items[:self.count])),
-            self.name + "_std": float(np.std(self.items[:self.count])),
-            self.name + "_quantiles": quantiles,
+            self.name + "_mean": float(np.nanmean(self.items[:self.count])),
+            self.name + "_std": float(np.nanstd(self.items[:self.count])),
+            self.name + "_quantiles": _quantiles,
         }