[rllib] Fix rollout.py with tuple action space (#5201)

* fix it

* update doc too

* fix rollout

python/ray/rllib/evaluation/episode.py

@@ -196,9 +196,6 @@ def _flatten_action(action):
     if isinstance(action, list) or isinstance(action, tuple):
         expanded = []
         for a in action:
-            if not hasattr(a, "shape") or len(a.shape) == 0:
-                expanded.append(np.expand_dims(a, 1))
-            else:
-                expanded.append(a)
+            expanded.append(np.reshape(a, [-1]))
         action = np.concatenate(expanded, axis=0).flatten()
     return action

python/ray/rllib/rollout.py

@@ -15,6 +15,7 @@ import ray
 from ray.rllib.agents.registry import get_agent_class
 from ray.rllib.env import MultiAgentEnv
 from ray.rllib.env.base_env import _DUMMY_AGENT_ID
+from ray.rllib.evaluation.episode import _flatten_action
 from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
 from ray.tune.util import merge_dicts
 
@@ -176,6 +177,7 @@ def rollout(agent, env_name, num_steps, out=None, no_render=True):
                             prev_action=prev_actions[agent_id],
                             prev_reward=prev_rewards[agent_id],
                             policy_id=policy_id)
+                    a_action = _flatten_action(a_action)  # tuple actions
                     action_dict[agent_id] = a_action
                     prev_actions[agent_id] = a_action
             action = action_dict