diff --git a/async_worker/ppo.py b/async_worker/ppo.py index 6d3f228..a576b2b 100644 --- a/async_worker/ppo.py +++ b/async_worker/ppo.py @@ -62,19 +62,15 @@ class ProximalPolicyOptimization: advantages = [] for i in range(config.rollout_length): mean, std, log_std = actor_net.predict(np.stack([state])) - # if not np.isfinite(mean.data.numpy()).all(): - # print('NaN', state, actor_net.predict(np.stack([state]))) + if not np.isfinite(mean.data.numpy()).all(): + print('NaN', state, actor_net.predict(np.stack([state]))) value = critic_net.predict(np.stack([state])) -<<<<<<< HEAD - # assert np.isfinite(mean.data.numpy().flatten()).all() - # assert np.isfinite(std.data.numpy().flatten()).all() - # action = self.policy.sample(mean.data.numpy().flatten(), std.data.numpy().flatten(), deterministic) + assert np.isfinite(mean.data.numpy().flatten()).all() + assert np.isfinite(std.data.numpy().flatten()).all() + action = self.policy.sample(mean.data.numpy().flatten(), std.data.numpy().flatten(), deterministic) action = self.policy.sample(mean.data.cpu().numpy().flatten(), std.data.cpu().numpy().flatten(), deterministic) - # assert np.isfinite(action).all() - # assert np.isfinite(value.data.numpy()).all() -======= - action = self.policy.sample(mean.data.cpu().numpy().flatten(), std.data.cpu().numpy().flatten(), deterministic) ->>>>>>> master + assert np.isfinite(action).all() + assert np.isfinite(value.data.numpy()).all() action = self.config.action_shift_fn(action) states.append(state) actions.append(action) @@ -88,7 +84,7 @@ class ProximalPolicyOptimization: episode_length += 1 reward = self.reward_normalizer(reward) - # assert np.isfinite(reward) + assert np.isfinite(reward) rewards.append(reward) # These seem to avoid NaN's I was getting that I couldn't replicate @@ -109,33 +105,23 @@ class ProximalPolicyOptimization: R = torch.zeros((1, 1)) if not done: R = critic_net.predict(np.stack([state])) - # assert np.isfinite(R.numpy()).all() + assert np.isfinite(R.numpy()).all() - -<<<<<<< HEAD values.append(critic_net.to_torch_variable(R)) A = critic_net.to_torch_variable(torch.zeros((1, 1))) discount = critic_net.to_torch_variable([self.config.discount]) gae_tau = critic_net.to_torch_variable([self.config.gae_tau]) for i in reversed(range(len(rewards))): R = critic_net.to_torch_variable([[rewards[i]]]) - # ret = R + self.config.discount * values[i + 1] -======= - values.append(actor_net.to_torch_variable(R)) - A = actor_net.to_torch_variable(torch.zeros((1, 1))) - discount = actor_net.to_torch_variable([self.config.discount]) - gae_tau = actor_net.to_torch_variable([self.config.gae_tau]) - for i in reversed(range(len(rewards))): - R = actor_net.to_torch_variable([[rewards[i]]]) ->>>>>>> master + ret = R + discount * values[i + 1] A = ret - values[i] + discount * gae_tau * A advantages.append(A.detach()) returns.append(ret.detach()) advantages = list(reversed(advantages)) returns = list(reversed(returns)) - # assert np.isfinite([a.data.numpy() for a in advantages]).all() - # assert np.isfinite([a.data.numpy() for a in returns]).all() + assert np.isfinite([a.data.numpy() for a in advantages]).all() + assert np.isfinite([a.data.numpy() for a in returns]).all() replay.feed([states, actions, returns, advantages]) batched_rewards /= batched_episode @@ -162,14 +148,14 @@ class ProximalPolicyOptimization: returns = torch.cat(returns, 0) advantages_raw = torch.cat(advantages, 0).squeeze(1) advantages = (advantages_raw - advantages_raw.mean()) / advantages_raw.std() - # assert np.isfinite(advantages.data.numpy()).all() - # assert np.isfinite(returns.data.numpy()).all() + assert np.isfinite(advantages.data.numpy()).all() + assert np.isfinite(returns.data.numpy()).all() config.logger.debug('sampled returns=%s advantages=%s advantages_raw=%s', returns[:10], advantages[:10], advantages_raw[:10]) mean_old, std_old, log_std_old = actor_net_old.predict(states) - # assert np.isfinite(mean_old.data.numpy()).all() - # assert np.isfinite(std_old.data.numpy()).all() - # assert np.isfinite(log_std_old.data.numpy()).all() + assert np.isfinite(mean_old.data.numpy()).all() + assert np.isfinite(std_old.data.numpy()).all() + assert np.isfinite(log_std_old.data.numpy()).all() probs_old = actor_net.log_density(actions, mean_old, log_std_old, std_old) mean, std, log_std = actor_net.predict(states) probs = actor_net.log_density(actions, mean, log_std, std) @@ -191,8 +177,8 @@ class ProximalPolicyOptimization: actor_net_old.load_state_dict(actor_net.state_dict()) self.worker_network.zero_grad() - # assert np.isfinite(value_loss.data.numpy()) - # assert np.isfinite(policy_loss.data.numpy()) + assert np.isfinite(value_loss.data.numpy()) + assert np.isfinite(policy_loss.data.numpy()) policy_loss.backward() value_loss.backward() config.logger.debug('policy_loss=%s value_loss=%s', policy_loss, value_loss) @@ -201,13 +187,7 @@ class ProximalPolicyOptimization: self.shared_network.zero_grad() self.actor_opt.zero_grad() self.critic_opt.zero_grad() -<<<<<<< HEAD - for param, worker_param in zip(self.shared_network.parameters(), self.worker_network.parameters()): - # assert np.isfinite(worker_param.grad.data.numpy()).all() - param._grad = worker_param.grad.clone() -======= sync_grad(self.shared_network, self.worker_network) ->>>>>>> master self.actor_opt.step() self.critic_opt.step() diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index 4ffccd2..013e0b4 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -159,20 +159,8 @@ class GaussianActorNet(nn.Module, BasicNet): return self.forward(x) def log_density(self, x, mean, log_std, std): -<<<<<<< HEAD - # x is action - # https://github.com/reinforceio/tensorforce/blob/master/tensorforce/core/distributions/gaussian.py#L85 - # same as tensorforce but max instead of + eps - sq_stddev = torch.max(std.pow(2), 1e-5) - sq_mean_distance = (x - mean).pow(2) - sq_stddev = tf.maximum(x=var, y=util.epsilon) - log_density = - 0.5 * sq_mean_distance / sq_stddev \ - - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x))\ - - log_std -======= var = std.pow(2) log_density = -(x - mean).pow(2) / (2 * var + 1e-5) - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x)) - log_std ->>>>>>> master return log_density.sum(1) def entropy(self, std):