From ec82fcbc478f50e5b14ea676e9f95b772a4ee66c Mon Sep 17 00:00:00 2001 From: Mike C Date: Mon, 22 Jan 2018 07:41:14 +0800 Subject: [PATCH] grad clip and tensorboard logging --- agent/DDPG_agent.py | 19 +++++++++++++++++++ async_worker/dpg.py | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 19f26d0..d012be5 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -64,6 +64,14 @@ class DDPGAgent: done = (done or (config.max_episode_length and steps >= config.max_episode_length)) next_state = self.state_normalizer(next_state) total_reward += reward + + # tensorboard logging + suffix = 'test_' if deterministic else '' + config.logger.scalar_summary(suffix+'action', action, self.total_steps) + config.logger.scalar_summary(suffix+'reward', reward, self.total_steps) + for key in info: + config.logger.scalar_summary('info_' + key, info[key], self.total_steps) + reward = self.reward_normalizer(reward) if not deterministic: @@ -92,6 +100,8 @@ class DDPGAgent: critic.zero_grad() self.critic_opt.zero_grad() critic_loss.backward() + if config.gradient_clip: + critic_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) self.critic_opt.step() actions = actor.predict(states, False) @@ -102,8 +112,17 @@ class DDPGAgent: actor.zero_grad() self.actor_opt.zero_grad() actions.backward(-var_actions.grad.data) + if config.gradient_clip: + actor_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) self.actor_opt.step() + # tensorboard logging + config.logger.scalar_summary('loss_policy', -var_actions.grad.data.sum(), self.total_steps) + config.logger.scalar_summary('loss_critic', critic_loss, self.total_steps) + if config.gradient_clip: + config.logger.histo_summary('grad_norm_actor', actor_grad_norm, self.total_steps) + config.logger.histo_summary('grad_norm_critic', critic_grad_norm, self.total_steps) + self.soft_update(self.target_network, self.worker_network) return total_reward, steps diff --git a/async_worker/dpg.py b/async_worker/dpg.py index 6137c25..eaa9272 100644 --- a/async_worker/dpg.py +++ b/async_worker/dpg.py @@ -61,6 +61,14 @@ class DeterministicPolicyGradient: done = (done or (config.max_episode_length and steps >= config.max_episode_length)) next_state = self.state_normalizer(next_state) total_reward += reward + + # tensorboard logging + suffix = 'test_' if deterministic else '' + config.logger.scalar_summary(suffix+'action', action, self.total_steps) + config.logger.scalar_summary(suffix+'reward', reward, self.total_steps) + for key in info: + config.logger.scalar_summary('info_' + key, info[key], self.total_steps) + reward = self.reward_normalizer(reward) if not deterministic: @@ -90,6 +98,8 @@ class DeterministicPolicyGradient: critic.zero_grad() self.critic_opt.zero_grad() critic_loss.backward() + if config.gradient_clip: + critic_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) with config.network_lock: sync_grad(self.shared_network.critic, critic) self.critic_opt.step() @@ -102,10 +112,19 @@ class DeterministicPolicyGradient: actor.zero_grad() self.actor_opt.zero_grad() actions.backward(-var_actions.grad.data) + if config.gradient_clip: + actor_grad_norm = nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) with config.network_lock: sync_grad(self.shared_network.actor, actor) self.actor_opt.step() + # tensorboard logging + config.logger.scalar_summary('loss_policy', -var_actions.grad.data, config.total_steps.value) + config.logger.scalar_summary('loss_critic', critic_loss, config.total_steps.value) + if config.gradient_clip: + config.logger.histo_summary('grad_norm_actor', actor_grad_norm, config.total_steps.value) + config.logger.histo_summary('grad_norm_critic', critic_grad_norm, config.total_steps.value) + self.worker_network.load_state_dict(self.shared_network.state_dict()) self.soft_update(self.target_network, self.worker_network)