wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-10 11:40:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge

Browse Source
This commit is contained in:
wassname
2017-11-15 17:17:55 +08:00
parent 0085cd7f72
commit 18edcfff9c
2 changed files with 19 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
async_worker/ppo.py
+19 -39
View File
@@ -62,19 +62,15 @@ class ProximalPolicyOptimization:
advantages = []
for i in range(config.rollout_length):
mean, std, log_std = actor_net.predict(np.stack([state]))
# if not np.isfinite(mean.data.numpy()).all():
# print('NaN', state, actor_net.predict(np.stack([state])))
if not np.isfinite(mean.data.numpy()).all():
print('NaN', state, actor_net.predict(np.stack([state])))
value = critic_net.predict(np.stack([state]))
<<<<<<< HEAD
# assert np.isfinite(mean.data.numpy().flatten()).all()
# assert np.isfinite(std.data.numpy().flatten()).all()
# action = self.policy.sample(mean.data.numpy().flatten(), std.data.numpy().flatten(), deterministic)
assert np.isfinite(mean.data.numpy().flatten()).all()
assert np.isfinite(std.data.numpy().flatten()).all()
action = self.policy.sample(mean.data.numpy().flatten(), std.data.numpy().flatten(), deterministic)
action = self.policy.sample(mean.data.cpu().numpy().flatten(), std.data.cpu().numpy().flatten(), deterministic)
# assert np.isfinite(action).all()
# assert np.isfinite(value.data.numpy()).all()
=======
action = self.policy.sample(mean.data.cpu().numpy().flatten(), std.data.cpu().numpy().flatten(), deterministic)
>>>>>>> master
assert np.isfinite(action).all()
assert np.isfinite(value.data.numpy()).all()
action = self.config.action_shift_fn(action)
states.append(state)
actions.append(action)
@@ -88,7 +84,7 @@ class ProximalPolicyOptimization:
episode_length += 1
reward = self.reward_normalizer(reward)
# assert np.isfinite(reward)
assert np.isfinite(reward)
rewards.append(reward)
# These seem to avoid NaN's I was getting that I couldn't replicate
@@ -109,33 +105,23 @@ class ProximalPolicyOptimization:
R = torch.zeros((1, 1))
if not done:
R = critic_net.predict(np.stack([state]))
# assert np.isfinite(R.numpy()).all()
assert np.isfinite(R.numpy()).all()
<<<<<<< HEAD
values.append(critic_net.to_torch_variable(R))
A = critic_net.to_torch_variable(torch.zeros((1, 1)))
discount = critic_net.to_torch_variable([self.config.discount])
gae_tau = critic_net.to_torch_variable([self.config.gae_tau])
for i in reversed(range(len(rewards))):
R = critic_net.to_torch_variable([[rewards[i]]])
# ret = R + self.config.discount * values[i + 1]
=======
values.append(actor_net.to_torch_variable(R))
A = actor_net.to_torch_variable(torch.zeros((1, 1)))
discount = actor_net.to_torch_variable([self.config.discount])
gae_tau = actor_net.to_torch_variable([self.config.gae_tau])
for i in reversed(range(len(rewards))):
R = actor_net.to_torch_variable([[rewards[i]]])
>>>>>>> master
ret = R + discount * values[i + 1]
A = ret - values[i] + discount * gae_tau * A
advantages.append(A.detach())
returns.append(ret.detach())
advantages = list(reversed(advantages))
returns = list(reversed(returns))
# assert np.isfinite([a.data.numpy() for a in advantages]).all()
# assert np.isfinite([a.data.numpy() for a in returns]).all()
assert np.isfinite([a.data.numpy() for a in advantages]).all()
assert np.isfinite([a.data.numpy() for a in returns]).all()
replay.feed([states, actions, returns, advantages])
batched_rewards /= batched_episode
@@ -162,14 +148,14 @@ class ProximalPolicyOptimization:
returns = torch.cat(returns, 0)
advantages_raw = torch.cat(advantages, 0).squeeze(1)
advantages = (advantages_raw - advantages_raw.mean()) / advantages_raw.std()
# assert np.isfinite(advantages.data.numpy()).all()
# assert np.isfinite(returns.data.numpy()).all()
assert np.isfinite(advantages.data.numpy()).all()
assert np.isfinite(returns.data.numpy()).all()
config.logger.debug('sampled returns=%s advantages=%s advantages_raw=%s', returns[:10], advantages[:10], advantages_raw[:10])
mean_old, std_old, log_std_old = actor_net_old.predict(states)
# assert np.isfinite(mean_old.data.numpy()).all()
# assert np.isfinite(std_old.data.numpy()).all()
# assert np.isfinite(log_std_old.data.numpy()).all()
assert np.isfinite(mean_old.data.numpy()).all()
assert np.isfinite(std_old.data.numpy()).all()
assert np.isfinite(log_std_old.data.numpy()).all()
probs_old = actor_net.log_density(actions, mean_old, log_std_old, std_old)
mean, std, log_std = actor_net.predict(states)
probs = actor_net.log_density(actions, mean, log_std, std)
@@ -191,8 +177,8 @@ class ProximalPolicyOptimization:
actor_net_old.load_state_dict(actor_net.state_dict())
self.worker_network.zero_grad()
# assert np.isfinite(value_loss.data.numpy())
# assert np.isfinite(policy_loss.data.numpy())
assert np.isfinite(value_loss.data.numpy())
assert np.isfinite(policy_loss.data.numpy())
policy_loss.backward()
value_loss.backward()
config.logger.debug('policy_loss=%s value_loss=%s', policy_loss, value_loss)
@@ -201,13 +187,7 @@ class ProximalPolicyOptimization:
self.shared_network.zero_grad()
self.actor_opt.zero_grad()
self.critic_opt.zero_grad()
<<<<<<< HEAD
for param, worker_param in zip(self.shared_network.parameters(), self.worker_network.parameters()):
# assert np.isfinite(worker_param.grad.data.numpy()).all()
param._grad = worker_param.grad.clone()
=======
sync_grad(self.shared_network, self.worker_network)
>>>>>>> master
self.actor_opt.step()
self.critic_opt.step()
network/continuous_action_network.py
-12
View File
@@ -159,20 +159,8 @@ class GaussianActorNet(nn.Module, BasicNet):
return self.forward(x)
def log_density(self, x, mean, log_std, std):
<<<<<<< HEAD
# x is action
# https://github.com/reinforceio/tensorforce/blob/master/tensorforce/core/distributions/gaussian.py#L85
# same as tensorforce but max instead of + eps
sq_stddev = torch.max(std.pow(2), 1e-5)
sq_mean_distance = (x - mean).pow(2)
sq_stddev = tf.maximum(x=var, y=util.epsilon)
log_density = - 0.5 * sq_mean_distance / sq_stddev \
- 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x))\
- log_std
=======
var = std.pow(2)
log_density = -(x - mean).pow(2) / (2 * var + 1e-5) - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x)) - log_std
>>>>>>> master
return log_density.sum(1)
def entropy(self, std):