From a5a5cf54b4f57e0377b2fc08461833b236a3ffe1 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Thu, 26 Apr 2018 11:53:07 -0600 Subject: [PATCH] Update DDPG parameters --- agent/DDPG_agent.py | 4 +--- main.py | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index b5590e0..a36243c 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -30,7 +30,6 @@ class DDPGAgent(BaseAgent): self.critic_opt = config.critic_optimizer_fn(self.critic.parameters()) self.replay = config.replay_fn() self.random_process = config.random_process_fn(self.task.action_dim) - self.criterion = nn.MSELoss() self.total_steps = 0 def soft_update(self, target, src): @@ -89,7 +88,7 @@ class DDPGAgent(BaseAgent): q_next.add_(rewards) q_next = q_next.detach() q = critic.predict(states, actions) - critic_loss = self.criterion(q, q_next) + critic_loss = (q - q_next).pow(2).mul(0.5).sum(-1).mean() self.critic_opt.zero_grad() critic_loss.backward() @@ -99,7 +98,6 @@ class DDPGAgent(BaseAgent): self.actor_opt.zero_grad() policy_loss.backward() - torch.nn.utils.clip_grad_value_(actor.parameters(), config.gradient_clip) self.actor_opt.step() self.soft_update(self.target_network, self.network) diff --git a/main.py b/main.py index e3bc505..ad1eee3 100644 --- a/main.py +++ b/main.py @@ -322,7 +322,7 @@ def ddpg_continuous(): config.critic_network_fn = lambda state_dim, action_dim: DeterministicCriticNet( TwoLayerFCBodyWithAction(state_dim, action_dim, [400, 300])) config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) - config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) + config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=64) config.discount = 0.99 config.state_normalizer = RunningStatsNormalizer() @@ -331,7 +331,6 @@ def ddpg_continuous(): n_steps_annealing=1000000) config.min_memory_size = 64 config.target_network_mix = 1e-3 - config.gradient_clip = 1.0 config.logger = Logger('./log', logger) run_episodes(DDPGAgent(config)) @@ -390,7 +389,7 @@ if __name__ == '__main__': # ppo_pixel_atari('BreakoutNoFrameskip-v4') # dqn_ram_atari('Breakout-ramNoFrameskip-v4') - # ddpg_continuous() + ddpg_continuous() # ppo_continuous() # action_conditional_video_prediction()