wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

temporary asserts to find NaNs

Browse Source
This commit is contained in:
wassname
2017-11-01 18:54:30 +08:00
parent 55ae92619a
commit ac9aa60f30
2 changed files with 60 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/DDPG_agent.py
+30 -2
View File
@@ -58,10 +58,12 @@ class DDPGAgent:
action += max(self.epsilon, config.min_epsilon) * self.random_process.sample()
self.epsilon -= self.d_epsilon
next_state, reward, done, info = self.task.step(action)
assert np.isfinite(reward)
done = (done or (config.max_episode_length and steps >= config.max_episode_length))
next_state = self.state_normalizer(next_state)
total_reward += reward
# reward = self.reward_normalizer(reward)
reward = self.reward_normalizer(reward) # I turned this one - Mik
assert np.isfinite(total_reward)
if not deterministic:
self.replay.feed([state, action, reward, next_state, int(done)])
@@ -76,6 +78,7 @@ class DDPGAgent:
self.learning_network.train()
experiences = self.replay.sample()
states, actions, rewards, next_states, terminals = experiences
assert np.isfinite(rewards).all()
q_next = target_critic.predict(next_states, target_actor.predict(next_states))
terminals = critic.to_torch_variable(terminals).unsqueeze(1)
rewards = critic.to_torch_variable(rewards).unsqueeze(1)
@@ -83,8 +86,26 @@ class DDPGAgent:
q_next.add_(rewards)
q_next = q_next.detach()
q = critic.predict(states, actions)
critic_loss = self.criterion(q, q_next)
# BUG Q blows up, it's wierd even thought when I calculate it
# I get e.g. [0.1,0.2,0.3], when I look at stored values it's
# [0.1,0.2,9e10] not sure why...
# So let's clip it for now
def clip(x, xmin, xmax):
x[x>xmax]=xmax
x[x<xmin]=xmin
return x
qmax=1e5
if np.abs(q.data.numpy()).max()>qmax:
config.logger.warning('q is above %s',qmax)
q = clip(q, -qmax, qmax)
q_next = clip(q_next, -qmax, qmax)
if np.abs(q_next.data.numpy()).max()>qmax:
config.logger.warning('q_next is above %s',qmax)
q = clip(q, -qmax, qmax)
q_next = clip(q_next, -qmax, qmax)
critic_loss = self.criterion(q, q_next)
assert np.isfinite(critic_loss.data.numpy())
critic.zero_grad()
critic_loss.backward()
self.critic_opt.step()
@@ -92,14 +113,21 @@ class DDPGAgent:
actions = actor.predict(states, False)
var_actions = Variable(actions.data, requires_grad=True)
q = critic.predict(states, var_actions)
critic.zero_grad() # is this something I need? Mike
q.backward(torch.ones(q.size()))
actor.zero_grad()
actions.backward(-var_actions.grad.data)
self.actor_opt.step()
config.logger.debug('-var_actions.grad.data: %s', -var_actions.grad.data)
config.logger.debug('q.size(): %s', q.size())
config.logger.debug('critic_loss: %s', critic_loss)
self.soft_update(self.target_network, self.learning_network)
q = None
q_next = None
return total_reward, steps
def save(self, file_name):
async_worker/ppo.py
+30 -5
View File
@@ -53,7 +53,6 @@ class ProximalPolicyOptimization:
actor_net_old.load_state_dict(actor_net.state_dict())
self.worker_network.load_state_dict(self.shared_network.state_dict())
while not replay.full():
states = []
actions = []
@@ -61,18 +60,22 @@ class ProximalPolicyOptimization:
values = []
returns = []
advantages = []
for i in range(config.rollout_length):
mean, std, log_std = actor_net.predict(np.stack([state]))
if not np.isfinite(mean.data.numpy()).all():
print('NaN', state, actor_net.predict(np.stack([state])))
value = critic_net.predict(np.stack([state]))
assert np.isfinite(mean.data.numpy().flatten()).all()
assert np.isfinite(std.data.numpy().flatten()).all()
action = self.policy.sample(mean.data.numpy().flatten(), std.data.numpy().flatten(), deterministic)
assert np.isfinite(action).all()
assert np.isfinite(value.data.numpy()).all()
action = self.config.action_shift_fn(action)
states.append(state)
actions.append(action)
values.append(value)
state, reward, done, _ = self.task.step(action)
state = self.state_normalizer(state)
# print state
done = (done or (config.max_episode_length and episode_length > config.max_episode_length))
batched_rewards += reward
@@ -80,8 +83,17 @@ class ProximalPolicyOptimization:
episode_length += 1
reward = self.reward_normalizer(reward)
assert np.isfinite(reward)
rewards.append(reward)
# These seem to avoid NaN's I was getting that I couldn't replicate
# even when debugging at the same point, and in the foreground
mean = None
std = None
log_std = None
value = None
action = None
if done:
episode_length = 0
batched_episode += 1
@@ -92,6 +104,7 @@ class ProximalPolicyOptimization:
R = torch.zeros((1, 1))
if not done:
R = critic_net.predict(np.stack([state])).data
assert np.isfinite(R.numpy()).all()
values.append(Variable(R))
A = Variable(torch.zeros((1, 1)))
@@ -103,6 +116,8 @@ class ProximalPolicyOptimization:
returns.append(ret.detach())
advantages = list(reversed(advantages))
returns = list(reversed(returns))
assert np.isfinite([a.data.numpy() for a in advantages]).all()
assert np.isfinite([a.data.numpy() for a in returns]).all()
replay.feed([states, actions, returns, advantages])
batched_rewards /= batched_episode
@@ -127,10 +142,16 @@ class ProximalPolicyOptimization:
states = actor_net.to_torch_variable(np.stack(states))
actions = actor_net.to_torch_variable(np.stack(actions))
returns = torch.cat(returns, 0)
advantages = torch.cat(advantages, 0).squeeze(1)
advantages = (advantages - advantages.mean()) / advantages.std()
advantages_raw = torch.cat(advantages, 0).squeeze(1)
advantages = (advantages_raw - advantages_raw.mean()) / advantages_raw.std()
assert np.isfinite(advantages.data.numpy()).all()
assert np.isfinite(returns.data.numpy()).all()
config.logger.debug('sampled returns=%s advantages=%s advantages_raw=%s', returns[:10], advantages[:10], advantages_raw[:10])
mean_old, std_old, log_std_old = actor_net_old.predict(states)
assert np.isfinite(mean_old.data.numpy()).all()
assert np.isfinite(std_old.data.numpy()).all()
assert np.isfinite(log_std_old.data.numpy()).all()
probs_old = actor_net.log_density(actions, mean_old, log_std_old, std_old)
mean, std, log_std = actor_net.predict(states)
probs = actor_net.log_density(actions, mean, log_std, std)
@@ -146,14 +167,18 @@ class ProximalPolicyOptimization:
actor_net_old.load_state_dict(actor_net.state_dict())
self.worker_network.zero_grad()
assert np.isfinite(value_loss.data.numpy())
assert np.isfinite(policy_loss.data.numpy())
policy_loss.backward()
value_loss.backward()
config.logger.debug('policy_loss=%s value_loss=%s', policy_loss, value_loss)
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
with config.network_lock:
self.shared_network.zero_grad()
self.actor_opt.zero_grad()
self.critic_opt.zero_grad()
for param, worker_param in zip(self.shared_network.parameters(), self.worker_network.parameters()):
assert np.isfinite(worker_param.grad.data.numpy()).all()
param._grad = worker_param.grad.clone()
self.actor_opt.step()
self.critic_opt.step()