mirror of
https://github.com/wassname/ray.git
synced 2026-07-19 11:27:32 +08:00
Eric LiangandPhilipp Moritz
4374ad1453
* add tf metrics * comments * fix network scopes * add doc * initial work * try with 3 virtual cpus * clean up metrics * use format string * fix trace level * back to pong * always run summary on cpu * plot intermediate and final sgd stats * add back a global step * update * add timeline * use staging area and reuse weights properly * stage at cpu * whoops, stage only the batch * clean up a bit * fix py flake * wip * create an optimizer graph per device * print timeline on 5th batch instead * print examples per second * log placement for training ops * force placement on cpu:0 * try separating weights onto different gpus * try using nccl * add cpu fallback * remove space from date * check has gpu device * fix flag config * checkpoint * wip * update * add some timing * trace loading * try cpu * revert that * remove expensive test * lint * cleanups * clean up timers * clean it up a bit * fix code for non-scalar action spaces * address some nits * fix quotes * efficient shuffling between sgd epochs
53 lines
2.0 KiB
Python
53 lines
2.0 KiB
Python
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import gym.spaces
|
|
import tensorflow as tf
|
|
from reinforce.models.visionnet import vision_net
|
|
from reinforce.models.fcnet import fc_net
|
|
|
|
|
|
class ProximalPolicyLoss(object):
|
|
|
|
def __init__(
|
|
self, observation_space, action_space,
|
|
observations, advantages, actions, prev_logits, logit_dim,
|
|
kl_coeff, distribution_class, config, sess):
|
|
assert (isinstance(action_space, gym.spaces.Discrete) or
|
|
isinstance(action_space, gym.spaces.Box))
|
|
self.prev_dist = distribution_class(prev_logits)
|
|
|
|
# Saved so that we can compute actions given different observations
|
|
self.observations = observations
|
|
|
|
if len(observation_space.shape) > 1:
|
|
self.curr_logits = vision_net(observations, num_classes=logit_dim)
|
|
else:
|
|
assert len(observation_space.shape) == 1
|
|
self.curr_logits = fc_net(observations, num_classes=logit_dim)
|
|
self.curr_dist = distribution_class(self.curr_logits)
|
|
self.sampler = self.curr_dist.sample()
|
|
|
|
# Make loss functions.
|
|
self.ratio = tf.exp(self.curr_dist.logp(actions) -
|
|
self.prev_dist.logp(actions))
|
|
self.kl = self.prev_dist.kl(self.curr_dist)
|
|
self.mean_kl = tf.reduce_mean(self.kl)
|
|
self.entropy = self.curr_dist.entropy()
|
|
self.mean_entropy = tf.reduce_mean(self.entropy)
|
|
self.surr1 = self.ratio * advantages
|
|
self.surr2 = tf.clip_by_value(self.ratio, 1 - config["clip_param"],
|
|
1 + config["clip_param"]) * advantages
|
|
self.surr = tf.minimum(self.surr1, self.surr2)
|
|
self.loss = tf.reduce_mean(-self.surr + kl_coeff * self.kl -
|
|
config["entropy_coeff"] * self.entropy)
|
|
self.sess = sess
|
|
|
|
def compute_actions(self, observations):
|
|
return self.sess.run([self.sampler, self.curr_logits],
|
|
feed_dict={self.observations: observations})
|
|
|
|
def loss(self):
|
|
return self.loss
|