From 4c6481d5beab36a736c732fbeebf0e3424f794ba Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Fri, 6 Apr 2018 10:34:52 -0600 Subject: [PATCH] Tune parameters for DDPG --- agent/DDPG_agent.py | 12 +++++++----- main.py | 10 +++++----- network/continuous_action_network.py | 26 ++++++++------------------ 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 36bb2e6..4aa01ee 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -29,8 +29,8 @@ class DDPGAgent(BaseAgent): self.criterion = nn.MSELoss() self.total_steps = 0 - self.state_normalizer = Normalizer(self.task.state_dim) - self.reward_normalizer = Normalizer(1) + # self.state_normalizer = Normalizer(self.task.state_dim) + # self.reward_normalizer = Normalizer(1) def soft_update(self, target, src): for target_param, param in zip(target.parameters(), src.parameters()): @@ -40,7 +40,7 @@ class DDPGAgent(BaseAgent): def episode(self, deterministic=False, video_recorder=None): self.random_process.reset_states() state = self.task.reset() - state = self.state_normalizer(state) + # state = self.state_normalizer(state) config = self.config actor = self.worker_network.actor @@ -59,9 +59,9 @@ class DDPGAgent(BaseAgent): next_state, reward, done, info = self.task.step(action) if video_recorder is not None: video_recorder.capture_frame() - next_state = self.state_normalizer(next_state) + # next_state = self.state_normalizer(next_state) total_reward += reward - reward = self.reward_normalizer(reward) + # reward = self.reward_normalizer(reward) if not deterministic: self.replay.feed([state, action, reward, next_state, int(done)]) @@ -99,6 +99,8 @@ class DDPGAgent(BaseAgent): actor.zero_grad() self.actor_opt.zero_grad() actions.backward(-var_actions.grad.data) + for param in actor.parameters(): + param.grad.data.clamp(-config.gradient_clip, config.gradient_clip) self.actor_opt.step() self.soft_update(self.target_network, self.worker_network) diff --git a/main.py b/main.py index e025dea..9c35374 100644 --- a/main.py +++ b/main.py @@ -240,15 +240,15 @@ def ddpg_continuous(): DisjointActorCriticNet(state_dim, action_dim, actor_network_fn, critic_network_fn) config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) config.critic_optimizer_fn =\ - lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) + lambda params: torch.optim.Adam(params, lr=1e-4) config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) config.discount = 0.99 config.random_process_fn = \ - lambda action_dim: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.2, + lambda action_dim: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.3, n_steps_annealing=100000) - config.min_memory_size = 50 - config.target_network_mix = 0.001 - config.render_episode_freq = 0 + config.min_memory_size = 64 + config.target_network_mix = 1e-3 + config.gradient_clip = 1.0 config.logger = Logger('./log', logger) run_episodes(DDPGAgent(config)) diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index 6e7f417..cfe9c11 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -13,12 +13,11 @@ class DeterministicActorNet(nn.Module, BasicNet): action_gate=F.tanh, action_scale=1, gpu=-1, - non_linear=F.tanh, - hidden_size=64): + non_linear=F.tanh): super(DeterministicActorNet, self).__init__() - self.layer1 = nn.Linear(state_dim, hidden_size) - self.layer2 = nn.Linear(hidden_size, hidden_size) - self.layer3 = nn.Linear(hidden_size, action_dim) + self.layer1 = nn.Linear(state_dim, 300) + self.layer2 = nn.Linear(300, 200) + self.layer3 = nn.Linear(200, action_dim) self.action_gate = action_gate self.action_scale = action_scale self.non_linear = non_linear @@ -54,12 +53,11 @@ class DeterministicCriticNet(nn.Module, BasicNet): state_dim, action_dim, gpu=-1, - non_linear=F.tanh, - hidden_size=64): + non_linear=F.tanh): super(DeterministicCriticNet, self).__init__() - self.layer1 = nn.Linear(state_dim, hidden_size) - self.layer2 = nn.Linear(hidden_size + action_dim, hidden_size) - self.layer3 = nn.Linear(hidden_size, 1) + self.layer1 = nn.Linear(state_dim, 400) + self.layer2 = nn.Linear(400 + action_dim, 300) + self.layer3 = nn.Linear(300, 1) self.non_linear = non_linear self.init_weights() BasicNet.__init__(self, gpu) @@ -122,14 +120,6 @@ class GaussianActorNet(nn.Module, BasicNet): def predict(self, x): return self.forward(x) - # def log_density(self, x, mean, log_std, std): - # var = std.pow(2) - # log_density = -(x - mean).pow(2) / (2 * var + 1e-5) - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x)) - log_std - # return log_density.sum(1) - # - # def entropy(self, std): - # return 0.5 * (1 + (2 * std.pow(2) * np.pi + 1e-5).log()).sum(1).mean() - class GaussianCriticNet(nn.Module, BasicNet): def __init__(self, state_dim,