From b161200f0fa3e0a826232ff0e654c47d51059855 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Wed, 2 Aug 2017 14:35:51 -0600 Subject: [PATCH] Add BN layer --- agent/DDPG_agent.py | 17 +++++++++++ main.py | 45 ++++++++++++++-------------- network/continuous_action_network.py | 30 +++++++++++++++---- 3 files changed, 64 insertions(+), 28 deletions(-) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 961947d..a5ecfac 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -8,6 +8,7 @@ from network import * from component import * from utils import * import pickle +import torch.nn as nn class DDPGAgent: def __init__(self, config): @@ -19,6 +20,8 @@ class DDPGAgent: self.target_critic = config.critic_network_fn() self.target_actor.load_state_dict(self.actor.state_dict()) self.target_critic.load_state_dict(self.critic.state_dict()) + self.target_actor.eval() + self.target_critic.eval() self.actor_opt = config.actor_optimizer_fn(self.actor.parameters()) self.critic_opt = config.critic_optimizer_fn(self.critic.parameters()) self.replay = config.replay_fn() @@ -41,8 +44,16 @@ class DDPGAgent: steps = 0 total_reward = 0.0 while not self.config or steps < self.config.max_episode_length: + self.actor.eval() action = self.actor.predict(np.stack([state])).flatten() + self.config.logger.histo_summary('state', state, self.total_steps) self.config.logger.histo_summary('action', action, self.total_steps) + self.config.logger.histo_summary('layer1_act', self.actor.layer1_act, self.total_steps) + self.config.logger.histo_summary('layer2_act', self.actor.layer2_act, self.total_steps) + self.config.logger.histo_summary('layer3_act', self.actor.layer3_act, self.total_steps) + self.config.logger.histo_summary('layer1_weight', self.actor.layer1_w, self.total_steps) + self.config.logger.histo_summary('layer2_weight', self.actor.layer2_w, self.total_steps) + self.config.logger.histo_summary('layer3_weight', self.actor.layer3_w, self.total_steps) if not deterministic: if self.total_steps < self.config.exploration_steps: action = self.task.random_action() @@ -50,6 +61,7 @@ class DDPGAgent: action += max(self.epsilon, 0) * self.random_process.sample() self.epsilon -= self.d_epsilon self.config.logger.histo_summary('noised action', action, self.total_steps) + action = self.config.action_shift_fn(action) next_state, reward, done, info = self.task.step(action) next_state = self.config.state_shift_fn(next_state) self.config.logger.scalar_summary('reward', reward, self.total_steps) @@ -65,6 +77,8 @@ class DDPGAgent: break if not deterministic and self.total_steps > self.config.exploration_steps: + self.actor.train() + self.critic.train() experiences = self.replay.sample() states, actions, rewards, next_states, terminals = experiences q_next = self.target_critic.predict(next_states, self.target_actor.predict(next_states)) @@ -85,6 +99,9 @@ class DDPGAgent: self.actor.zero_grad() actor_loss.backward() + self.config.logger.histo_summary('layer1_g', self.actor.layer1.weight.grad.data.numpy(), self.total_steps) + self.config.logger.histo_summary('layer2_g', self.actor.layer2.weight.grad.data.numpy(), self.total_steps) + self.config.logger.histo_summary('layer3_g', self.actor.layer3.weight.grad.data.numpy(), self.total_steps) self.actor_opt.step() self.soft_update(self.target_actor, self.actor) diff --git a/main.py b/main.py index e1dd212..be4e02f 100644 --- a/main.py +++ b/main.py @@ -198,35 +198,36 @@ def ddpg_pendulum(): config.noise_decay_interval = 10000 config.random_process_fn = \ lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) - config.test_interval = 10 + config.test_interval = 0 config.test_repetitions = 10 config.logger = Logger('./log', gym.logger) agent = DDPGAgent(config) agent.run() -def ddpg_bipedal_walker(): +def ddpg_walker(): task_fn = lambda: BipedalWalker() task = task_fn() - config = dict() - config['task_fn'] = task_fn - config['actor_network_fn'] = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, gpu=True) - config['critic_network_fn'] = lambda: DDPGCriticNet(task.state_dim, task.action_dim, gpu=True) - config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4) - config['critic_optimizer_fn'] =\ + config = Config() + config.task_fn = task_fn + # shifter = Shifter() + # config.state_shift_fn = lambda state: shifter(state) + config.actor_network_fn = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, 1, gpu=True) + config.critic_network_fn = lambda: DDPGCriticNet(task.state_dim, task.action_dim, gpu=True) + config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) + config.critic_optimizer_fn =\ lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) - config['replay_fn'] = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) - config['discount'] = 0.99 - config['step_limit'] = 1000 - config['tau'] = 0.001 - config['exploration_steps'] = 100 - config['noise_decay_steps'] = 10000 - config['random_process_fn'] = \ + config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) + config.discount = 0.99 + config.max_episode_length = 1000 + config.target_network_mix = 0.001 + config.exploration_steps = 100 + config.noise_decay_interval = 10000 + config.random_process_fn = \ lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) - config['test_interval'] = 10 - config['test_repetitions'] = 10 - config['tag'] = '' - config['logger'] = Logger('./log', gym.logger, True) - agent = DDPGAgent(**config) + config.test_interval = 0 + config.test_repetitions = 5 + config.logger = Logger('./log', gym.logger) + agent = DDPGAgent(config) agent.run() if __name__ == '__main__': @@ -238,7 +239,8 @@ if __name__ == '__main__': # a3c_cart_pole() # a3c_pendulum() # a3c_walker() - ddpg_pendulum() + # ddpg_pendulum() + ddpg_walker() # dqn_pixel_atari('PongNoFrameskip-v3') # async_pixel_atari('PongNoFrameskip-v3') @@ -248,4 +250,3 @@ if __name__ == '__main__': # async_pixel_atari('BreakoutNoFrameskip-v3') # a3c_pixel_atari('BreakoutNoFrameskip-v3') - # ddpg_bipedal_walker() \ No newline at end of file diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index 95e01ba..e6a7a88 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -51,9 +51,13 @@ class DDPGActorNet(nn.Module, BasicNet): action_scale, gpu=False): super(DDPGActorNet, self).__init__() - self.layer1 = nn.Linear(state_dim, 400) - self.layer2 = nn.Linear(400, 300) - self.layer3 = nn.Linear(300, action_dim) + hidden1 = 400 + hidden2 = 300 + self.layer1 = nn.Linear(state_dim, hidden1) + self.bn1 = nn.BatchNorm1d(hidden1) + self.layer2 = nn.Linear(hidden1, hidden2) + self.bn2 = nn.BatchNorm1d(hidden2) + self.layer3 = nn.Linear(hidden2, action_dim) self.action_gate = action_gate self.action_scale = action_scale BasicNet.__init__(self, None, False, False) @@ -76,8 +80,16 @@ class DDPGActorNet(nn.Module, BasicNet): def forward(self, x): x = self.to_torch_variable(x) x = F.relu(self.layer1(x)) + self.layer1_w = self.layer1.weight.data.cpu().numpy() + self.layer1_act = x.data.cpu().numpy() + x = self.bn1(x) x = F.relu(self.layer2(x)) + self.layer2_w = self.layer2.weight.data.cpu().numpy() + self.layer2_act = x.data.cpu().numpy() + x = self.bn2(x) x = self.layer3(x) + self.layer3_w = self.layer3.weight.data.cpu().numpy() + self.layer3_act = x.data.cpu().numpy() x = self.action_scale * self.action_gate(x) return x @@ -93,9 +105,13 @@ class DDPGCriticNet(nn.Module, BasicNet): action_dim, gpu=False): super(DDPGCriticNet, self).__init__() - self.layer1 = nn.Linear(state_dim, 400) - self.layer2 = nn.Linear(400 + action_dim, 300) - self.layer3 = nn.Linear(300, 1) + hidden1 = 400 + hidden2 = 300 + self.layer1 = nn.Linear(state_dim, hidden1) + self.bn1 = nn.BatchNorm1d(hidden1) + self.layer2 = nn.Linear(hidden1 + action_dim, hidden2) + self.bn2 = nn.BatchNorm1d(hidden2) + self.layer3 = nn.Linear(hidden2, 1) BasicNet.__init__(self, None, False, False) self.init_weights() @@ -117,7 +133,9 @@ class DDPGCriticNet(nn.Module, BasicNet): x = self.to_torch_variable(x) action = self.to_torch_variable(action) x = F.relu(self.layer1(x)) + x = self.bn1(x) x = F.relu(self.layer2(torch.cat([x, action], dim=1))) + x = self.bn2(x) x = self.layer3(x) return x