Multiagent model using concatenated observations (#1416)

* working multi action distribution and multiagent model

* currently working but the splits arent done in the right place

* added shared models

* added categorical support and mountain car example

* now compatible with generalized advantage estimation

* working multiagent code with discrete and continuous example

* moved reshaper to utils

* code review changes made, ppo action placeholder moved to model catalog, all multiagent code moved out of fcnet

* added examples in

* added PEP8 compliance

* examples are mostly pep8 compliant

* removed all flake errors

* added examples to jenkins tests

* fixed custom options bug

* added lines to let docker file find multiagent tests

* shortened example run length

* corrected nits

* fixed flake errors

python/ray/rllib/examples/__init__.py

@@ -0,0 +1,5 @@
+# flake8: noqa
+from ray.rllib.examples.multiagent_mountaincar_env \
+    import MultiAgentMountainCarEnv
+from ray.rllib.examples.multiagent_pendulum_env \
+    import MultiAgentPendulumEnv

python/ray/rllib/examples/multiagent_mountaincar.py

@@ -0,0 +1,56 @@
+""" Multiagent mountain car. Each agent outputs an action which
+is summed to form the total action. This is a discrete
+multiagent example
+"""
+
+import gym
+from gym.envs.registration import register
+
+import ray
+import ray.rllib.ppo as ppo
+from ray.tune.registry import get_registry, register_env
+
+env_name = "MultiAgentMountainCarEnv"
+
+env_version_num = 0
+env_name = env_name + '-v' + str(env_version_num)
+
+
+def pass_params_to_gym(env_name):
+    global env_version_num
+
+    register(
+      id=env_name,
+      entry_point='ray.rllib.examples:' + "MultiAgentMountainCarEnv",
+      max_episode_steps=200,
+      kwargs={}
+    )
+
+
+def create_env(env_config):
+    pass_params_to_gym(env_name)
+    env = gym.envs.make(env_name)
+    return env
+
+
+if __name__ == '__main__':
+    register_env(env_name, lambda env_config: create_env(env_config))
+    config = ppo.DEFAULT_CONFIG.copy()
+    horizon = 200
+    num_cpus = 2
+    ray.init(num_cpus=num_cpus, redirect_output=False)
+    config["num_workers"] = num_cpus
+    config["timesteps_per_batch"] = 100
+    config["num_sgd_iter"] = 10
+    config["gamma"] = 0.999
+    config["horizon"] = horizon
+    config["use_gae"] = True
+    config["model"].update({"fcnet_hiddens": [256, 256]})
+    options = {"multiagent_obs_shapes": [2, 2],
+               "multiagent_act_shapes": [3, 3],
+               "multiagent_shared_model": False,
+               "multiagent_fcnet_hiddens": [[32, 32]] * 2}
+    config["model"].update({"custom_options": options})
+    alg = ppo.PPOAgent(env=env_name, registry=get_registry(), config=config)
+    for i in range(1):
+        alg.train()

python/ray/rllib/examples/multiagent_mountaincar_env.py

@@ -0,0 +1,52 @@
+import math
+from gym.spaces import Box, Tuple, Discrete
+import numpy as np
+from gym.envs.classic_control.mountain_car import MountainCarEnv
+
+"""
+Multiagent mountain car that sums and then
+averages its actions to produce the velocity
+"""
+
+
+class MultiAgentMountainCarEnv(MountainCarEnv):
+    def __init__(self):
+        self.min_position = -1.2
+        self.max_position = 0.6
+        self.max_speed = 0.07
+        self.goal_position = 0.5
+
+        self.low = np.array([self.min_position, -self.max_speed])
+        self.high = np.array([self.max_position, self.max_speed])
+
+        self.viewer = None
+
+        self.action_space = [Discrete(3) for _ in range(2)]
+        self.observation_space = Tuple(tuple(Box(self.low, self.high)
+                                             for _ in range(2)))
+
+        self._seed()
+        self.reset()
+
+    def _step(self, action):
+        summed_act = 0.5 * np.sum(action)
+
+        position, velocity = self.state
+        velocity += (summed_act - 1) * 0.001
+        velocity += math.cos(3 * position) * (-0.0025)
+        velocity = np.clip(velocity, -self.max_speed, self.max_speed)
+        position += velocity
+        position = np.clip(position, self.min_position, self.max_position)
+        if (position == self.min_position and velocity < 0):
+            velocity = 0
+
+        done = bool(position >= self.goal_position)
+
+        reward = position
+
+        self.state = (position, velocity)
+        return [np.array(self.state) for _ in range(2)], reward, done, {}
+
+    def _reset(self):
+        self.state = np.array([self.np_random.uniform(low=-0.6, high=-0.4), 0])
+        return [np.array(self.state) for _ in range(2)]

python/ray/rllib/examples/multiagent_pendulum.py

@@ -0,0 +1,56 @@
+""" Run script for multiagent pendulum env. Each agent outputs a
+torque which is summed to form the total torque. This is a
+continuous multiagent example
+"""
+
+import gym
+from gym.envs.registration import register
+
+import ray
+import ray.rllib.ppo as ppo
+from ray.tune.registry import get_registry, register_env
+
+env_name = "MultiAgentPendulumEnv"
+
+env_version_num = 0
+env_name = env_name + '-v' + str(env_version_num)
+
+
+def pass_params_to_gym(env_name):
+    global env_version_num
+
+    register(
+      id=env_name,
+      entry_point='ray.rllib.examples:' + "MultiAgentPendulumEnv",
+      max_episode_steps=100,
+      kwargs={}
+    )
+
+
+def create_env(env_config):
+    pass_params_to_gym(env_name)
+    env = gym.envs.make(env_name)
+    return env
+
+
+if __name__ == '__main__':
+    register_env(env_name, lambda env_config: create_env(env_config))
+    config = ppo.DEFAULT_CONFIG.copy()
+    horizon = 100
+    num_cpus = 2
+    ray.init(num_cpus=num_cpus, redirect_output=False)
+    config["num_workers"] = num_cpus
+    config["timesteps_per_batch"] = 100
+    config["num_sgd_iter"] = 10
+    config["gamma"] = 0.999
+    config["horizon"] = horizon
+    config["use_gae"] = True
+    config["model"].update({"fcnet_hiddens": [256, 256]})
+    options = {"multiagent_obs_shapes": [3, 3],
+               "multiagent_act_shapes": [1, 1],
+               "multiagent_shared_model": True,
+               "multiagent_fcnet_hiddens": [[32, 32]] * 2}
+    config["model"].update({"custom_options": options})
+    alg = ppo.PPOAgent(env=env_name, registry=get_registry(), config=config)
+    for i in range(1):
+        alg.train()

python/ray/rllib/examples/multiagent_pendulum_env.py

@@ -0,0 +1,70 @@
+from gym.spaces import Box, Tuple
+from gym.utils import seeding
+from gym.envs.classic_control.pendulum import PendulumEnv
+import numpy as np
+
+"""
+ Multiagent pendulum that sums its torques to generate an action
+"""
+
+
+class MultiAgentPendulumEnv(PendulumEnv):
+    metadata = {
+      'render.modes': ['human', 'rgb_array'],
+      'video.frames_per_second': 30
+    }
+
+    def __init__(self):
+        self.max_speed = 8
+        self.max_torque = 2.
+        self.dt = .05
+        self.viewer = None
+
+        high = np.array([1., 1., self.max_speed])
+        self.action_space = [Box(low=-self.max_torque / 2,
+                                 high=self.max_torque / 2, shape=(1,))
+                             for _ in range(2)]
+        self.observation_space = Tuple(tuple(Box(low=-high, high=high)
+                                             for _ in range(2)))
+
+        self._seed()
+
+    def _seed(self, seed=None):
+        self.np_random, seed = seeding.np_random(seed)
+        return [seed]
+
+    def _step(self, u):
+        th, thdot = self.state  # th := theta
+
+        summed_u = np.sum(u)
+        g = 10.
+        m = 1.
+        length = 1.
+        dt = self.dt
+
+        summed_u = np.clip(summed_u, -self.max_torque, self.max_torque)
+        self.last_u = summed_u  # for rendering
+        costs = self.angle_normalize(th) ** 2 + .1 * thdot ** 2 + \
+            .001 * (summed_u ** 2)
+
+        newthdot = thdot + (-3 * g / (2 * length) * np.sin(th + np.pi) +
+                            3. / (m * length ** 2) * summed_u) * dt
+        newth = th + newthdot * dt
+        newthdot = np.clip(newthdot, -self.max_speed, self.max_speed)
+
+        self.state = np.array([newth, newthdot])
+        return self._get_obs(), -costs, False, {}
+
+    def _reset(self):
+        high = np.array([np.pi, 1])
+        self.state = self.np_random.uniform(low=-high, high=high)
+        self.last_u = None
+        return self._get_obs()
+
+    def _get_obs(self):
+        theta, thetadot = self.state
+        return [np.array([np.cos(theta), np.sin(theta), thetadot])
+                for _ in range(2)]
+
+    def angle_normalize(self, x):
+        return (((x + np.pi) % (2 * np.pi)) - np.pi)