[rllib] Add custom value functions, fix up and document multi-agent variable sharing (#3151)

python/ray/rllib/models/model.py

@@ -7,6 +7,7 @@ from collections import OrderedDict
 import gym
 import tensorflow as tf
 
+from ray.rllib.models.misc import linear, normc_initializer
 from ray.rllib.models.preprocessors import get_preprocessor
 
 
@@ -131,6 +132,18 @@ class Model(object):
         """
         raise NotImplementedError
 
+    def value_function(self):
+        """Builds the value function output.
+
+        This method can be overridden to customize the implementation of the
+        value function (e.g., not sharing hidden layers).
+
+        Returns:
+            Tensor of size [BATCH_SIZE] for the value function.
+        """
+        return tf.reshape(
+            linear(self.last_layer, 1, "value", normc_initializer(1.0)), [-1])
+
 
 def _restore_original_dimensions(input_dict, obs_space):
     if hasattr(obs_space, "original_space"):