wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-08-29 11:12:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

grad clip and tensorboard logging

Browse Source
This commit is contained in:
Mike C
2018-01-22 07:41:14 +08:00
parent 51c800afb4
commit ec82fcbc47
2 changed files with 38 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/DDPG_agent.py
+19
View File
@@ -64,6 +64,14 @@ class DDPGAgent:
done = (done or (config.max_episode_length and steps >= config.max_episode_length))
next_state = self.state_normalizer(next_state)
total_reward += reward
# tensorboard logging
suffix = 'test_' if deterministic else ''
config.logger.scalar_summary(suffix+'action', action, self.total_steps)
config.logger.scalar_summary(suffix+'reward', reward, self.total_steps)
for key in info:
config.logger.scalar_summary('info_' + key, info[key], self.total_steps)
reward = self.reward_normalizer(reward)
if not deterministic:
@@ -92,6 +100,8 @@ class DDPGAgent:
critic.zero_grad()
self.critic_opt.zero_grad()
critic_loss.backward()
if config.gradient_clip:
critic_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
self.critic_opt.step()
actions = actor.predict(states, False)
@@ -102,8 +112,17 @@ class DDPGAgent:
actor.zero_grad()
self.actor_opt.zero_grad()
actions.backward(-var_actions.grad.data)
if config.gradient_clip:
actor_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
self.actor_opt.step()
# tensorboard logging
config.logger.scalar_summary('loss_policy', -var_actions.grad.data.sum(), self.total_steps)
config.logger.scalar_summary('loss_critic', critic_loss, self.total_steps)
if config.gradient_clip:
config.logger.histo_summary('grad_norm_actor', actor_grad_norm, self.total_steps)
config.logger.histo_summary('grad_norm_critic', critic_grad_norm, self.total_steps)
self.soft_update(self.target_network, self.worker_network)
return total_reward, steps
async_worker/dpg.py
+19
View File
@@ -61,6 +61,14 @@ class DeterministicPolicyGradient:
done = (done or (config.max_episode_length and steps >= config.max_episode_length))
next_state = self.state_normalizer(next_state)
total_reward += reward
# tensorboard logging
suffix = 'test_' if deterministic else ''
config.logger.scalar_summary(suffix+'action', action, self.total_steps)
config.logger.scalar_summary(suffix+'reward', reward, self.total_steps)
for key in info:
config.logger.scalar_summary('info_' + key, info[key], self.total_steps)
reward = self.reward_normalizer(reward)
if not deterministic:
@@ -90,6 +98,8 @@ class DeterministicPolicyGradient:
critic.zero_grad()
self.critic_opt.zero_grad()
critic_loss.backward()
if config.gradient_clip:
critic_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
with config.network_lock:
sync_grad(self.shared_network.critic, critic)
self.critic_opt.step()
@@ -102,10 +112,19 @@ class DeterministicPolicyGradient:
actor.zero_grad()
self.actor_opt.zero_grad()
actions.backward(-var_actions.grad.data)
if config.gradient_clip:
actor_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
with config.network_lock:
sync_grad(self.shared_network.actor, actor)
self.actor_opt.step()
# tensorboard logging
config.logger.scalar_summary('loss_policy', -var_actions.grad.data, config.total_steps.value)
config.logger.scalar_summary('loss_critic', critic_loss, config.total_steps.value)
if config.gradient_clip:
config.logger.histo_summary('grad_norm_actor', actor_grad_norm, config.total_steps.value)
config.logger.histo_summary('grad_norm_critic', critic_grad_norm, config.total_steps.value)
self.worker_network.load_state_dict(self.shared_network.state_dict())
self.soft_update(self.target_network, self.worker_network)