[rllib] throw an error for continuous action spaces in IMPALA

We currently don't support this since the reference vtrace.py does not, though it could be an interesting extension.

python/ray/rllib/agents/impala/vtrace_policy_graph.py

@@ -107,10 +107,7 @@ class VTracePolicyGraph(LearningRateSchedule, TFPolicyGraph):
                                           tf.get_variable_scope().name)
 
         # Setup the policy loss
-        if isinstance(action_space, gym.spaces.Box):
-            ac_size = action_space.shape[0]
-            actions = tf.placeholder(tf.float32, [None, ac_size], name="ac")
-        elif isinstance(action_space, gym.spaces.Discrete):
+        if isinstance(action_space, gym.spaces.Discrete):
             ac_size = action_space.n
             actions = tf.placeholder(tf.int64, [None], name="ac")
         else:

python/ray/rllib/test/test_supported_spaces.py

@@ -94,6 +94,7 @@ class ModelSupportedSpaces(unittest.TestCase):
     def testAll(self):
         ray.init()
         stats = {}
+        check_support("IMPALA", {"gpu": False}, stats)
         check_support("DDPG", {"timesteps_per_iteration": 1}, stats)
         check_support("DQN", {"timesteps_per_iteration": 1}, stats)
         check_support("A3C", {