[RLlib] MADDPG bug fix (issue https://github.com/ray-project/ray/issues/8483) (#9110)

* Bug fix for https://githhub.com/ray-project/ray/issues/8483

We need to pass in a framework explicitly with the new defaults. Further, the actual bug was that policies were being sorted alphabetically in the MADDPG init(), which led to incorrect initialization.

* Linting

rllib/contrib/maddpg/maddpg.py

@@ -23,6 +23,9 @@ logger.setLevel(logging.INFO)
 # yapf: disable
 # __sphinx_doc_begin__
 DEFAULT_CONFIG = with_common_config({
+    # === Framework to run the algorithm ===
+    "framework": "tf",
+
     # === Settings for each individual policy ===
     # ID of the agent controlled by this policy
     "agent_id": None,

rllib/contrib/maddpg/maddpg_policy.py

@@ -74,12 +74,14 @@ class MADDPGTFPolicy(MADDPGPostprocessing, TFPolicy):
                     "Space {} is not supported.".format(space))
 
         obs_space_n = [
-            _make_continuous_space(space) for _, (_, space, _, _) in
-            sorted(config["multiagent"]["policies"].items())
+            _make_continuous_space(space)
+            for _, (_, space, _,
+                    _) in config["multiagent"]["policies"].items()
         ]
         act_space_n = [
-            _make_continuous_space(space) for _, (_, _, space, _) in
-            sorted(config["multiagent"]["policies"].items())
+            _make_continuous_space(space)
+            for _, (_, _, space,
+                    _) in config["multiagent"]["policies"].items()
         ]
 
         # _____ Placeholders