Multiagent model using concatenated observations (#1416)

* working multi action distribution and multiagent model

* currently working but the splits arent done in the right place

* added shared models

* added categorical support and mountain car example

* now compatible with generalized advantage estimation

* working multiagent code with discrete and continuous example

* moved reshaper to utils

* code review changes made, ppo action placeholder moved to model catalog, all multiagent code moved out of fcnet

* added examples in

* added PEP8 compliance

* examples are mostly pep8 compliant

* removed all flake errors

* added examples to jenkins tests

* fixed custom options bug

* added lines to let docker file find multiagent tests

* shortened example run length

* corrected nits

* fixed flake errors

python/ray/rllib/models/action_dist.py

@@ -4,6 +4,7 @@ from __future__ import print_function
 
 import tensorflow as tf
 import numpy as np
+from ray.rllib.utils.reshaper import Reshaper
 
 
 class ActionDistribution(object):
@@ -109,3 +110,49 @@ class Deterministic(ActionDistribution):
 
     def sample(self):
         return self.inputs
+
+
+class MultiActionDistribution(ActionDistribution):
+    """Action distribution that operates for list of actions.
+
+    Args:
+        inputs (Tensor list): A list of tensors from which to compute samples.
+    """
+    def __init__(self, inputs, action_space, child_distributions):
+        # you actually have to instantiate the child distributions
+        self.reshaper = Reshaper(action_space)
+        split_inputs = self.reshaper.split_tensor(inputs)
+        child_list = []
+        for i, distribution in enumerate(child_distributions):
+            child_list.append(distribution(split_inputs[i]))
+        self.child_distributions = child_list
+
+    def logp(self, x):
+        """The log-likelihood of the action distribution."""
+        split_list = self.reshaper.split_tensor(x)
+        for i, distribution in enumerate(self.child_distributions):
+            # Remove extra categorical dimension
+            if isinstance(distribution, Categorical):
+                split_list[i] = tf.squeeze(split_list[i], axis=-1)
+        log_list = np.asarray([distribution.logp(split_x) for
+                              distribution, split_x in
+                               zip(self.child_distributions, split_list)])
+        return np.sum(log_list)
+
+    def kl(self, other):
+        """The KL-divergence between two action distributions."""
+        kl_list = np.asarray([distribution.kl(other_distribution) for
+                              distribution, other_distribution in
+                              zip(self.child_distributions,
+                                  other.child_distributions)])
+        return np.sum(kl_list)
+
+    def entropy(self):
+        """The entropy of the action distribution."""
+        entropy_list = np.array([s.entropy() for s in
+                                 self.child_distributions])
+        return np.sum(entropy_list)
+
+    def sample(self):
+        """Draw a sample from the action distribution."""
+        return [[s.sample() for s in self.child_distributions]]