From c5cbc10f94f0e9230dcd19aa128ce7cfba65f7fc Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Fri, 6 Oct 2017 11:16:21 -0600 Subject: [PATCH] Refactor DDPG --- agent/DDPG_agent.py | 83 +++++++++---------- async_worker/continuous_actor_critic.py | 9 -- component/task.py | 16 ++++ main.py | 49 +++++++++-- network/continuous_action_network.py | 106 ++++++++++++++---------- utils/config.py | 3 +- utils/normalizer.py | 11 +++ 7 files changed, 167 insertions(+), 110 deletions(-) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index a5ecfac..7bf4ad2 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -14,16 +14,12 @@ class DDPGAgent: def __init__(self, config): self.config = config self.task = config.task_fn() - self.actor = config.actor_network_fn() - self.critic = config.critic_network_fn() - self.target_actor = config.actor_network_fn() - self.target_critic = config.critic_network_fn() - self.target_actor.load_state_dict(self.actor.state_dict()) - self.target_critic.load_state_dict(self.critic.state_dict()) - self.target_actor.eval() - self.target_critic.eval() - self.actor_opt = config.actor_optimizer_fn(self.actor.parameters()) - self.critic_opt = config.critic_optimizer_fn(self.critic.parameters()) + self.learning_network = config.network_fn() + self.target_network = config.network_fn() + self.target_network.load_state_dict(self.learning_network.state_dict()) + self.target_network.eval() + self.actor_opt = config.actor_optimizer_fn(self.learning_network.actor.parameters()) + self.critic_opt = config.critic_optimizer_fn(self.learning_network.critic.parameters()) self.replay = config.replay_fn() self.random_process = config.random_process_fn() self.criterion = nn.MSELoss() @@ -31,6 +27,9 @@ class DDPGAgent: self.epsilon = 1.0 self.d_epsilon = 1.0 / config.noise_decay_interval + self.state_normalizer = Normalizer(self.task.state_dim) + self.reward_normalizer = Normalizer(1) + def soft_update(self, target, src): for target_param, param in zip(target.parameters(), src.parameters()): target_param.data.copy_(target_param.data * (1.0 - self.config.target_network_mix) + @@ -39,34 +38,31 @@ class DDPGAgent: def episode(self, deterministic=False): self.random_process.reset_states() state = self.task.reset() - state = self.config.state_shift_fn(state) + state = self.state_normalizer(state) + + config = self.config + actor = self.learning_network.actor + critic = self.learning_network.critic + target_actor = self.target_network.actor + target_critic = self.target_network.critic steps = 0 total_reward = 0.0 - while not self.config or steps < self.config.max_episode_length: - self.actor.eval() - action = self.actor.predict(np.stack([state])).flatten() - self.config.logger.histo_summary('state', state, self.total_steps) - self.config.logger.histo_summary('action', action, self.total_steps) - self.config.logger.histo_summary('layer1_act', self.actor.layer1_act, self.total_steps) - self.config.logger.histo_summary('layer2_act', self.actor.layer2_act, self.total_steps) - self.config.logger.histo_summary('layer3_act', self.actor.layer3_act, self.total_steps) - self.config.logger.histo_summary('layer1_weight', self.actor.layer1_w, self.total_steps) - self.config.logger.histo_summary('layer2_weight', self.actor.layer2_w, self.total_steps) - self.config.logger.histo_summary('layer3_weight', self.actor.layer3_w, self.total_steps) + while True: + actor.eval() + action = actor.predict(np.stack([state])).flatten() if not deterministic: - if self.total_steps < self.config.exploration_steps: + if self.total_steps < config.exploration_steps: action = self.task.random_action() else: - action += max(self.epsilon, 0) * self.random_process.sample() + action += max(self.epsilon, config.min_epsilon) * self.random_process.sample() self.epsilon -= self.d_epsilon - self.config.logger.histo_summary('noised action', action, self.total_steps) - action = self.config.action_shift_fn(action) next_state, reward, done, info = self.task.step(action) - next_state = self.config.state_shift_fn(next_state) - self.config.logger.scalar_summary('reward', reward, self.total_steps) + done = (done or (config.max_episode_length and steps >= config.max_episode_length)) + next_state = self.state_normalizer(next_state) total_reward += reward - reward = self.config.reward_shift_fn(reward) + reward = np.asscalar(self.reward_normalizer(np.array([reward]))) + if not deterministic: self.replay.feed([state, action, reward, next_state, int(done)]) self.total_steps += 1 @@ -76,36 +72,31 @@ class DDPGAgent: if done: break - if not deterministic and self.total_steps > self.config.exploration_steps: - self.actor.train() - self.critic.train() + if not deterministic and self.total_steps > config.exploration_steps: + self.learning_network.train() experiences = self.replay.sample() states, actions, rewards, next_states, terminals = experiences - q_next = self.target_critic.predict(next_states, self.target_actor.predict(next_states)) - terminals = self.critic.to_torch_variable(terminals).unsqueeze(1) - rewards = self.critic.to_torch_variable(rewards).unsqueeze(1) - q_next = self.config.discount * q_next * (1 - terminals) + q_next = target_critic.predict(next_states, target_actor.predict(next_states)) + terminals = critic.to_torch_variable(terminals).unsqueeze(1) + rewards = critic.to_torch_variable(rewards).unsqueeze(1) + q_next = config.discount * q_next * (1 - terminals) q_next.add_(rewards) - q_next = Variable(q_next.data) - q = self.critic.predict(states, actions) + q_next = q_next.detach() + q = critic.predict(states, actions) critic_loss = self.criterion(q, q_next) - self.critic.zero_grad() + critic.zero_grad() critic_loss.backward() self.critic_opt.step() - actor_loss = -self.critic.predict(states, self.actor.predict(states, False)) + actor_loss = -critic.predict(states, actor.predict(states, False)) actor_loss = actor_loss.mean() - self.actor.zero_grad() + actor.zero_grad() actor_loss.backward() - self.config.logger.histo_summary('layer1_g', self.actor.layer1.weight.grad.data.numpy(), self.total_steps) - self.config.logger.histo_summary('layer2_g', self.actor.layer2.weight.grad.data.numpy(), self.total_steps) - self.config.logger.histo_summary('layer3_g', self.actor.layer3.weight.grad.data.numpy(), self.total_steps) self.actor_opt.step() - self.soft_update(self.target_actor, self.actor) - self.soft_update(self.target_critic, self.critic) + self.soft_update(self.target_network, self.learning_network) return total_reward diff --git a/async_worker/continuous_actor_critic.py b/async_worker/continuous_actor_critic.py index 68c5468..16a4152 100644 --- a/async_worker/continuous_actor_critic.py +++ b/async_worker/continuous_actor_critic.py @@ -46,15 +46,6 @@ class ContinuousAdvantageActorCritic: terminal = (terminal or (config.max_episode_length and steps >= config.max_episode_length)) next_state = self.state_normalizer(next_state) - # if deterministic: - # self.config.logger.scalar_summary('reward', reward, self.counter) - # self.config.logger.histo_summary('std', std.data.numpy(), self.counter) - # self.config.logger.histo_summary('mean', mean.data.numpy(), self.counter) - # self.config.logger.histo_summary('action', action, self.counter) - # self.config.logger.scalar_summary('steps', steps, self.counter) - # self.config.logger.histo_summary('states', state, self.counter) - # self.counter += 1 - steps += 1 total_reward += reward reward = np.asscalar(self.reward_normalizer(np.array([reward]))) diff --git a/component/task.py b/component/task.py index 394a329..a6eb803 100644 --- a/component/task.py +++ b/component/task.py @@ -108,6 +108,22 @@ class BipedalWalker(BasicTask): next_state, reward, done, info = self.env.step(action) return next_state, reward, done, info +class ContinuousLunarLander(BasicTask): + name = 'LunarLanderContinuous-v2' + success_threshold = 300 + + def __init__(self): + BasicTask.__init__(self) + self.env = gym.make(self.name) + self.env._max_episode_steps = sys.maxsize + self.action_dim = self.env.action_space.shape[0] + self.state_dim = self.env.observation_space.shape[0] + + def step(self, action): + action = np.clip(action, -1, 1) + next_state, reward, done, info = self.env.step(action) + return next_state, reward, done, info + class Fruit(BasicTask): def __init__(self, hybrid_reward=False, pseudo_reward=False, atomic_state=True): self.hybrid_reward = hybrid_reward diff --git a/main.py b/main.py index 16bcac4..00e4d97 100644 --- a/main.py +++ b/main.py @@ -181,8 +181,11 @@ def ddpg_pendulum(): task = task_fn() config = Config() config.task_fn = task_fn - config.actor_network_fn = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, 2) - config.critic_network_fn = lambda: DDPGCriticNet(task.state_dim, task.action_dim) + config.actor_network_fn = lambda: DeterministicActorNet( + task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.tanh) + config.critic_network_fn = lambda: DeterministicCriticNet( + task.state_dim, task.action_dim, non_linear=F.tanh) + config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) config.critic_optimizer_fn =\ lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) @@ -200,15 +203,14 @@ def ddpg_pendulum(): agent = DDPGAgent(config) agent.run() -def ddpg_walker(): - task_fn = lambda: BipedalWalker() +def ddpg_lunar_lander(): + task_fn = lambda: ContinuousLunarLander() task = task_fn() config = Config() config.task_fn = task_fn - # shifter = Shifter() - # config.state_shift_fn = lambda state: shifter(state) - config.actor_network_fn = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, 1, gpu=True) - config.critic_network_fn = lambda: DDPGCriticNet(task.state_dim, task.action_dim, gpu=True) + config.actor_network_fn = lambda: DeterministicActorNet(task.state_dim, task.action_dim, F.tanh, 1) + config.critic_network_fn = lambda: DeterministicCriticNet(task.state_dim, task.action_dim) + config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) config.critic_optimizer_fn =\ lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) @@ -221,6 +223,34 @@ def ddpg_walker(): config.random_process_fn = \ lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) config.test_interval = 0 + config.test_repetitions = 10 + config.logger = Logger('./log', gym.logger) + agent = DDPGAgent(config) + agent.run() + +def ddpg_walker(): + task_fn = lambda: BipedalWalker() + task = task_fn() + config = Config() + config.task_fn = task_fn + config.actor_network_fn = lambda: DeterministicActorNet( + task.state_dim, task.action_dim, F.tanh, 1, gpu=True, batch_norm=False, non_linear=F.tanh) + config.critic_network_fn = lambda: DeterministicCriticNet( + task.state_dim, task.action_dim, gpu=True, batch_norm=False, non_linear=F.tanh) + config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) + config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) + config.critic_optimizer_fn =\ + lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) + config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) + config.discount = 0.99 + config.min_epsilon = 0.1 + config.max_episode_length = 999 + config.target_network_mix = 0.001 + config.exploration_steps = 10000 + config.noise_decay_interval = 1000000 + config.random_process_fn = \ + lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) + config.test_interval = 0 config.test_repetitions = 5 config.logger = Logger('./log', gym.logger) agent = DDPGAgent(config) @@ -341,8 +371,9 @@ if __name__ == '__main__': # a3c_pendulum() # a3c_walker() # ddpg_pendulum() + ddpg_lunar_lander() # ddpg_walker() - ppo_pendulum() + # ppo_pendulum() # dqn_fruit() # hrdqn_fruit() diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index 3a6e292..ff53b77 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -6,53 +6,55 @@ from network import * -class DDPGActorNet(nn.Module, BasicNet): +class DeterministicActorNet(nn.Module, BasicNet): def __init__(self, state_dim, action_dim, action_gate, action_scale, - gpu=False): - super(DDPGActorNet, self).__init__() - hidden1 = 400 - hidden2 = 300 - self.layer1 = nn.Linear(state_dim, hidden1) - self.bn1 = nn.BatchNorm1d(hidden1) - self.layer2 = nn.Linear(hidden1, hidden2) - self.bn2 = nn.BatchNorm1d(hidden2) - self.layer3 = nn.Linear(hidden2, action_dim) + gpu=False, + batch_norm=False, + non_linear=F.relu): + super(DeterministicActorNet, self).__init__() + hidden_size = 64 + self.layer1 = nn.Linear(state_dim, hidden_size) + self.layer3 = nn.Linear(hidden_size, action_dim) self.action_gate = action_gate self.action_scale = action_scale - BasicNet.__init__(self, None, False, False) - self.init_weights() + self.non_linear = non_linear + + if batch_norm: + self.bn1 = nn.BatchNorm1d(hidden_size) + self.bn2 = nn.BatchNorm1d(hidden_size) + self.layer2 = nn.Linear(hidden_size, hidden_size) + + self.batch_norm = batch_norm + BasicNet.__init__(self, None, gpu, False) + # self.init_weights() def init_weights(self): bound = 3e-3 self.layer3.weight.data.uniform_(-bound, bound) - # self.layer3.bias.data.uniform_(-bound, bound) + self.layer3.bias.data.fill_(0) def fanin(size): v = 1.0 / np.sqrt(size[1]) return torch.FloatTensor(size).uniform_(-v, v) self.layer1.weight.data = fanin(self.layer1.weight.data.size()) - # self.layer1.bias.data = fanin(self.layer1.bias.data.size()) + self.layer1.bias.data.fill_(0) self.layer2.weight.data = fanin(self.layer2.weight.data.size()) - # self.layer2.bias.data = fanin(self.layer2.bias.data.size()) + self.layer2.bias.data.fill_(0) def forward(self, x): x = self.to_torch_variable(x) - x = F.relu(self.layer1(x)) - self.layer1_w = self.layer1.weight.data.cpu().numpy() - self.layer1_act = x.data.cpu().numpy() - x = self.bn1(x) - x = F.relu(self.layer2(x)) - self.layer2_w = self.layer2.weight.data.cpu().numpy() - self.layer2_act = x.data.cpu().numpy() - x = self.bn2(x) + x = self.non_linear(self.layer1(x)) + if self.batch_norm: + x = self.bn1(x) + x = self.non_linear(self.layer2(x)) + if self.batch_norm: + x = self.bn2(x) x = self.layer3(x) - self.layer3_w = self.layer3.weight.data.cpu().numpy() - self.layer3_act = x.data.cpu().numpy() x = self.action_scale * self.action_gate(x) return x @@ -62,43 +64,51 @@ class DDPGActorNet(nn.Module, BasicNet): y = y.cpu().data.numpy() return y -class DDPGCriticNet(nn.Module, BasicNet): +class DeterministicCriticNet(nn.Module, BasicNet): def __init__(self, state_dim, action_dim, - gpu=False): - super(DDPGCriticNet, self).__init__() - hidden1 = 400 - hidden2 = 300 - self.layer1 = nn.Linear(state_dim, hidden1) - self.bn1 = nn.BatchNorm1d(hidden1) - self.layer2 = nn.Linear(hidden1 + action_dim, hidden2) - self.bn2 = nn.BatchNorm1d(hidden2) - self.layer3 = nn.Linear(hidden2, 1) - BasicNet.__init__(self, None, False, False) - self.init_weights() + gpu=False, + batch_norm=False, + non_linear=F.relu): + super(DeterministicCriticNet, self).__init__() + hidden_size = 64 + self.layer1 = nn.Linear(state_dim, hidden_size) + self.layer2 = nn.Linear(hidden_size + action_dim, hidden_size) + self.layer3 = nn.Linear(hidden_size, 1) + self.non_linear = non_linear + + if batch_norm: + self.bn1 = nn.BatchNorm1d(hidden_size) + self.bn2 = nn.BatchNorm1d(hidden_size) + self.batch_norm = batch_norm + + BasicNet.__init__(self, None, gpu, False) + # self.init_weights() def init_weights(self): bound = 3e-3 self.layer3.weight.data.uniform_(-bound, bound) - # self.layer3.bias.data.uniform_(-bound, bound) + self.layer3.bias.data.fill_(0) def fanin(size): v = 1.0 / np.sqrt(size[1]) return torch.FloatTensor(size).uniform_(-v, v) self.layer1.weight.data = fanin(self.layer1.weight.data.size()) - # self.layer1.bias.data = fanin(self.layer1.bias.data.size()) + self.layer1.bias.data.fill_(0) self.layer2.weight.data = fanin(self.layer2.weight.data.size()) - # self.layer2.bias.data = fanin(self.layer2.bias.data.size()) + self.layer2.bias.data.fill_(0) def forward(self, x, action): x = self.to_torch_variable(x) action = self.to_torch_variable(action) - x = F.relu(self.layer1(x)) - x = self.bn1(x) - x = F.relu(self.layer2(torch.cat([x, action], dim=1))) - x = self.bn2(x) + x = self.non_linear(self.layer1(x)) + if self.batch_norm: + x = self.bn1(x) + x = self.non_linear(self.layer2(torch.cat([x, action], dim=1))) + if self.batch_norm: + x = self.bn2(x) x = self.layer3(x) return x @@ -191,3 +201,11 @@ class DisjointActorCriticNet: def zero_grad(self): self.actor.zero_grad() self.critic.zero_grad() + + def train(self): + self.actor.train() + self.critic.train() + + def eval(self): + self.actor.eval() + self.critic.eval() diff --git a/utils/config.py b/utils/config.py index 186ef61..32374a2 100644 --- a/utils/config.py +++ b/utils/config.py @@ -36,8 +36,6 @@ class Config: self.gae_tau = 1.0 self.noise_decay_interval = 0 self.target_network_mix = 0.001 - self.reward_shift_fn = lambda r: r - self.state_shift_fn = lambda s: s self.action_shift_fn = lambda a: a self.reward_weight = 1 self.hybrid_reward = False @@ -47,3 +45,4 @@ class Config: self.master_fn = None self.master_optimizer_fn = None self.num_heads = 10 + self.min_epsilon = 0 diff --git a/utils/normalizer.py b/utils/normalizer.py index 000e4e7..b32ddd2 100644 --- a/utils/normalizer.py +++ b/utils/normalizer.py @@ -5,6 +5,17 @@ ####################################################################### import torch +class Normalizer: + def __init__(self, o_size): + self.stats = SharedStats(o_size) + + def __call__(self, o_): + o = torch.FloatTensor(o_) + self.stats.feed(o) + std = (self.stats.v + 1e-6) ** .5 + o = (o - self.stats.m) / std + return o.numpy().reshape(o_.shape) + class StaticNormalizer: def __init__(self, o_size): self.offline_stats = SharedStats(o_size)