[RLlib] Examples folder restructuring (Model examples; final part). (#8278)

- This PR completes any previously missing PyTorch Model counterparts to TFModels in examples/models.
- It also makes sure, all example scripts in the rllib/examples folder are tested for both frameworks and learn the given task (this is often currently not checked) using a --as-test flag in connection with a --stop-reward.

rllib/examples/multi_agent_two_trainers.py

@@ -12,19 +12,27 @@ import argparse
 import gym
 
 import ray
-from ray.rllib.agents.dqn.dqn import DQNTrainer
-from ray.rllib.agents.dqn.dqn_tf_policy import DQNTFPolicy
-from ray.rllib.agents.ppo.ppo import PPOTrainer
-from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy
+from ray.rllib.agents.dqn import DQNTrainer, DQNTFPolicy, DQNTorchPolicy
+from ray.rllib.agents.ppo import PPOTrainer, PPOTFPolicy, PPOTorchPolicy
 from ray.rllib.examples.env.multi_agent import MultiAgentCartPole
 from ray.tune.logger import pretty_print
 from ray.tune.registry import register_env
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--num-iters", type=int, default=20)
+# Use torch for both policies.
+parser.add_argument("--torch", action="store_true")
+# Mix PPO=tf and DQN=torch if set.
+parser.add_argument("--mixed-torch-tf", action="store_true")
+parser.add_argument("--as-test", action="store_true")
+parser.add_argument("--stop-iters", type=int, default=20)
+parser.add_argument("--stop-reward", type=float, default=50)
+parser.add_argument("--stop-timesteps", type=int, default=100000)
 
 if __name__ == "__main__":
     args = parser.parse_args()
+    assert not (args.torch and args.mixed_torch_tf),\
+        "Use either --torch or --mixed-torch-tf, not both!"
+
     ray.init()
 
     # Simple environment with 4 independent cartpole entities
@@ -37,8 +45,10 @@ if __name__ == "__main__":
     # You can also have multiple policies per trainer, but here we just
     # show one each for PPO and DQN.
     policies = {
-        "ppo_policy": (PPOTFPolicy, obs_space, act_space, {}),
-        "dqn_policy": (DQNTFPolicy, obs_space, act_space, {}),
+        "ppo_policy": (PPOTorchPolicy if args.torch else PPOTFPolicy,
+                       obs_space, act_space, {}),
+        "dqn_policy": (DQNTorchPolicy if args.torch or args.mixed_torch_tf else
+                       DQNTFPolicy, obs_space, act_space, {}),
     }
 
     def policy_mapping_fn(agent_id):
@@ -59,6 +69,7 @@ if __name__ == "__main__":
             # disable filters, otherwise we would need to synchronize those
             # as well to the DQN agent
             "observation_filter": "NoFilter",
+            "use_pytorch": args.torch,
         })
 
     dqn_trainer = DQNTrainer(
@@ -71,6 +82,7 @@ if __name__ == "__main__":
             },
             "gamma": 0.95,
             "n_step": 3,
+            "use_pytorch": args.torch or args.mixed_torch_tf,
         })
 
     # You should see both the printed X and Y approach 200 as this trains:
@@ -78,17 +90,31 @@ if __name__ == "__main__":
     #   policy_reward_mean:
     #     dqn_policy: X
     #     ppo_policy: Y
-    for i in range(args.num_iters):
+    for i in range(args.stop_iters):
         print("== Iteration", i, "==")
 
         # improve the DQN policy
         print("-- DQN --")
-        print(pretty_print(dqn_trainer.train()))
+        result_dqn = dqn_trainer.train()
+        print(pretty_print(result_dqn))
 
         # improve the PPO policy
         print("-- PPO --")
-        print(pretty_print(ppo_trainer.train()))
+        result_ppo = ppo_trainer.train()
+        print(pretty_print(result_ppo))
+
+        # Test passed gracefully.
+        if args.as_test and \
+                result_dqn["episode_reward_mean"] > args.stop_reward and \
+                result_ppo["episode_reward_mean"] > args.stop_reward:
+            print("test passed (both agents above requested reward)")
+            quit(0)
 
         # swap weights to synchronize
         dqn_trainer.set_weights(ppo_trainer.get_weights(["ppo_policy"]))
         ppo_trainer.set_weights(dqn_trainer.get_weights(["dqn_policy"]))
+
+    # Desired reward not reached.
+    if args.as_test:
+        raise ValueError("Desired reward ({}) not reached!".format(
+            args.stop_reward))