From ac9aa60f30057a8b2c18726119921dee8ed1cda0 Mon Sep 17 00:00:00 2001 From: wassname Date: Wed, 1 Nov 2017 18:54:30 +0800 Subject: [PATCH] temporary asserts to find NaNs --- agent/DDPG_agent.py | 32 ++++++++++++++++++++++++++++++-- async_worker/ppo.py | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 2c71050..f0a3cc9 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -58,10 +58,12 @@ class DDPGAgent: action += max(self.epsilon, config.min_epsilon) * self.random_process.sample() self.epsilon -= self.d_epsilon next_state, reward, done, info = self.task.step(action) + assert np.isfinite(reward) done = (done or (config.max_episode_length and steps >= config.max_episode_length)) next_state = self.state_normalizer(next_state) total_reward += reward - # reward = self.reward_normalizer(reward) + reward = self.reward_normalizer(reward) # I turned this one - Mik + assert np.isfinite(total_reward) if not deterministic: self.replay.feed([state, action, reward, next_state, int(done)]) @@ -76,6 +78,7 @@ class DDPGAgent: self.learning_network.train() experiences = self.replay.sample() states, actions, rewards, next_states, terminals = experiences + assert np.isfinite(rewards).all() q_next = target_critic.predict(next_states, target_actor.predict(next_states)) terminals = critic.to_torch_variable(terminals).unsqueeze(1) rewards = critic.to_torch_variable(rewards).unsqueeze(1) @@ -83,8 +86,26 @@ class DDPGAgent: q_next.add_(rewards) q_next = q_next.detach() q = critic.predict(states, actions) - critic_loss = self.criterion(q, q_next) + # BUG Q blows up, it's wierd even thought when I calculate it + # I get e.g. [0.1,0.2,0.3], when I look at stored values it's + # [0.1,0.2,9e10] not sure why... + # So let's clip it for now + def clip(x, xmin, xmax): + x[x>xmax]=xmax + x[xqmax: + config.logger.warning('q is above %s',qmax) + q = clip(q, -qmax, qmax) + q_next = clip(q_next, -qmax, qmax) + if np.abs(q_next.data.numpy()).max()>qmax: + config.logger.warning('q_next is above %s',qmax) + q = clip(q, -qmax, qmax) + q_next = clip(q_next, -qmax, qmax) + critic_loss = self.criterion(q, q_next) + assert np.isfinite(critic_loss.data.numpy()) critic.zero_grad() critic_loss.backward() self.critic_opt.step() @@ -92,14 +113,21 @@ class DDPGAgent: actions = actor.predict(states, False) var_actions = Variable(actions.data, requires_grad=True) q = critic.predict(states, var_actions) + critic.zero_grad() # is this something I need? Mike q.backward(torch.ones(q.size())) actor.zero_grad() actions.backward(-var_actions.grad.data) self.actor_opt.step() + config.logger.debug('-var_actions.grad.data: %s', -var_actions.grad.data) + config.logger.debug('q.size(): %s', q.size()) + config.logger.debug('critic_loss: %s', critic_loss) self.soft_update(self.target_network, self.learning_network) + q = None + q_next = None + return total_reward, steps def save(self, file_name): diff --git a/async_worker/ppo.py b/async_worker/ppo.py index 9224c93..200a1a9 100644 --- a/async_worker/ppo.py +++ b/async_worker/ppo.py @@ -53,7 +53,6 @@ class ProximalPolicyOptimization: actor_net_old.load_state_dict(actor_net.state_dict()) self.worker_network.load_state_dict(self.shared_network.state_dict()) - while not replay.full(): states = [] actions = [] @@ -61,18 +60,22 @@ class ProximalPolicyOptimization: values = [] returns = [] advantages = [] - for i in range(config.rollout_length): mean, std, log_std = actor_net.predict(np.stack([state])) + if not np.isfinite(mean.data.numpy()).all(): + print('NaN', state, actor_net.predict(np.stack([state]))) value = critic_net.predict(np.stack([state])) + assert np.isfinite(mean.data.numpy().flatten()).all() + assert np.isfinite(std.data.numpy().flatten()).all() action = self.policy.sample(mean.data.numpy().flatten(), std.data.numpy().flatten(), deterministic) + assert np.isfinite(action).all() + assert np.isfinite(value.data.numpy()).all() action = self.config.action_shift_fn(action) states.append(state) actions.append(action) values.append(value) state, reward, done, _ = self.task.step(action) state = self.state_normalizer(state) - # print state done = (done or (config.max_episode_length and episode_length > config.max_episode_length)) batched_rewards += reward @@ -80,8 +83,17 @@ class ProximalPolicyOptimization: episode_length += 1 reward = self.reward_normalizer(reward) + assert np.isfinite(reward) rewards.append(reward) + # These seem to avoid NaN's I was getting that I couldn't replicate + # even when debugging at the same point, and in the foreground + mean = None + std = None + log_std = None + value = None + action = None + if done: episode_length = 0 batched_episode += 1 @@ -92,6 +104,7 @@ class ProximalPolicyOptimization: R = torch.zeros((1, 1)) if not done: R = critic_net.predict(np.stack([state])).data + assert np.isfinite(R.numpy()).all() values.append(Variable(R)) A = Variable(torch.zeros((1, 1))) @@ -103,6 +116,8 @@ class ProximalPolicyOptimization: returns.append(ret.detach()) advantages = list(reversed(advantages)) returns = list(reversed(returns)) + assert np.isfinite([a.data.numpy() for a in advantages]).all() + assert np.isfinite([a.data.numpy() for a in returns]).all() replay.feed([states, actions, returns, advantages]) batched_rewards /= batched_episode @@ -127,10 +142,16 @@ class ProximalPolicyOptimization: states = actor_net.to_torch_variable(np.stack(states)) actions = actor_net.to_torch_variable(np.stack(actions)) returns = torch.cat(returns, 0) - advantages = torch.cat(advantages, 0).squeeze(1) - advantages = (advantages - advantages.mean()) / advantages.std() + advantages_raw = torch.cat(advantages, 0).squeeze(1) + advantages = (advantages_raw - advantages_raw.mean()) / advantages_raw.std() + assert np.isfinite(advantages.data.numpy()).all() + assert np.isfinite(returns.data.numpy()).all() + config.logger.debug('sampled returns=%s advantages=%s advantages_raw=%s', returns[:10], advantages[:10], advantages_raw[:10]) mean_old, std_old, log_std_old = actor_net_old.predict(states) + assert np.isfinite(mean_old.data.numpy()).all() + assert np.isfinite(std_old.data.numpy()).all() + assert np.isfinite(log_std_old.data.numpy()).all() probs_old = actor_net.log_density(actions, mean_old, log_std_old, std_old) mean, std, log_std = actor_net.predict(states) probs = actor_net.log_density(actions, mean, log_std, std) @@ -146,14 +167,18 @@ class ProximalPolicyOptimization: actor_net_old.load_state_dict(actor_net.state_dict()) self.worker_network.zero_grad() + assert np.isfinite(value_loss.data.numpy()) + assert np.isfinite(policy_loss.data.numpy()) policy_loss.backward() value_loss.backward() + config.logger.debug('policy_loss=%s value_loss=%s', policy_loss, value_loss) nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) with config.network_lock: self.shared_network.zero_grad() self.actor_opt.zero_grad() self.critic_opt.zero_grad() for param, worker_param in zip(self.shared_network.parameters(), self.worker_network.parameters()): + assert np.isfinite(worker_param.grad.data.numpy()).all() param._grad = worker_param.grad.clone() self.actor_opt.step() self.critic_opt.step()