diff --git a/python/ray/rllib/evaluation/rollout_worker.py b/python/ray/rllib/evaluation/rollout_worker.py
index 489c21c08..410718c0e 100644
--- a/python/ray/rllib/evaluation/rollout_worker.py
+++ b/python/ray/rllib/evaluation/rollout_worker.py
@@ -750,10 +750,11 @@ def _validate_multiagent_config(policy, allow_none_graph=False):
         if not isinstance(k, str):
             raise ValueError("policy keys must be strs, got {}".format(
                 type(k)))
-        if not isinstance(v, tuple) or len(v) != 4:
+        if not isinstance(v, (tuple, list)) or len(v) != 4:
             raise ValueError(
-                "policy values must be tuples of "
-                "(cls, obs_space, action_space, config), got {}".format(v))
+                "policy values must be tuples/lists of "
+                "(cls or None, obs_space, action_space, config), got {}".
+                format(v))
         if allow_none_graph and v[0] is None:
             pass
         elif not issubclass(v[0], Policy):