wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-25 13:30:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[rllib] Add test for multi-agent support and fix IMPALA multi-agent (#3289)

Browse Source 
IMPALA support for multiagent was broken since IMPALA has a requirement that batch sizes be of a certain length. However multi-agent envs can create variable-length batches.

Fix this by adding zero-padding as needed (similar to the RNN case).
This commit is contained in:
Eric Liang
2018-11-14 14:14:07 -08:00
committed by GitHub
parent 57c7b4238e
commit 706dc1d473
15 changed files with 181 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/rllib/models/model.py
+14 -12
View File
@@ -62,6 +62,7 @@ class Model(object):
self.seq_lens = tf.placeholder(
dtype=tf.int32, shape=[None], name="seq_lens")
self._num_outputs = num_outputs
if options.get("free_log_std"):
assert num_outputs % 2 == 0
num_outputs = num_outputs // 2
@@ -73,18 +74,6 @@ class Model(object):
self.outputs, self.last_layer = self._build_layers(
input_dict["obs"], num_outputs, options)
# Validate the output shape
try:
out = tf.convert_to_tensor(self.outputs)
shape = out.shape.as_list()
except Exception:
raise ValueError("Output is not a tensor: {}".format(self.outputs))
else:
if len(shape) != 2 or shape[1] != num_outputs:
raise ValueError(
"Expected output shape of [None, {}], got {}".format(
num_outputs, shape))
if options.get("free_log_std", False):
log_std = tf.get_variable(
name="log_std",
@@ -93,6 +82,19 @@ class Model(object):
self.outputs = tf.concat(
[self.outputs, 0.0 * self.outputs + log_std], 1)
def _validate_output_shape(self):
"""Checks that the model has the correct number of outputs."""
try:
out = tf.convert_to_tensor(self.outputs)
shape = out.shape.as_list()
except Exception:
raise ValueError("Output is not a tensor: {}".format(self.outputs))
else:
if len(shape) != 2 or shape[1] != self._num_outputs:
raise ValueError(
"Expected output shape of [None, {}], got {}".format(
self._num_outputs, shape))
def _build_layers(self, inputs, num_outputs, options):
"""Builds and returns the output and last layer of the network.