wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-17 11:50:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

clip critic and actor seperately

Browse Source
This commit is contained in:
Mike C
2018-01-22 13:52:01 +08:00
parent 48e7d3f767
commit b8518c655d
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/DDPG_agent.py
+6 -2
View File
@@ -115,24 +115,28 @@ class DDPGAgent:
self.critic_opt.zero_grad()
critic_loss.backward()
if config.gradient_clip:
critic_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
critic_grad_norm = nn.utils.clip_grad_norm(self.worker_network.critic.parameters(), config.gradient_clip)
self.critic_opt.step()
actions = actor.predict(states, False)
var_actions = Variable(actions.data, requires_grad=True)
q = critic.predict(states, var_actions)
q.backward(torch.ones(q.size()))
critic.zero_grad()
actor.zero_grad()
self.actor_opt.zero_grad()
actions.backward(-var_actions.grad.data)
if config.gradient_clip:
actor_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
actor_grad_norm = nn.utils.clip_grad_norm(self.worker_network.actor.parameters(), config.gradient_clip)
self.actor_opt.step()
actor.zero_grad()
# tensorboard logging
config.logger.scalar_summary('loss_policy', -var_actions.grad.data.sum(), self.total_steps)
config.logger.scalar_summary('loss_critic', critic_loss, self.total_steps)
config.logger.scalar_summary('lr_actor', torch.FloatTensor([self.actor_opt.param_groups[0]['lr']]), self.total_steps)
config.logger.scalar_summary('lr_critic', torch.FloatTensor([self.critic_opt.param_groups[0]['lr']]), self.total_steps)
if config.gradient_clip:
config.logger.histo_summary('grad_norm_actor', actor_grad_norm, self.total_steps)
config.logger.histo_summary('grad_norm_critic', critic_grad_norm, self.total_steps)