diff --git a/rllib/rollout.py b/rllib/rollout.py
index 415cd869d..dfc599160 100755
--- a/rllib/rollout.py
+++ b/rllib/rollout.py
@@ -451,7 +451,8 @@ def rollout(agent,
 
             if multiagent:
                 done = done["__all__"]
-                reward_total += sum(reward.values())
+                reward_total += sum(
+                    r for r in reward.values() if r is not None)
             else:
                 reward_total += reward
             if not no_render: