From 5116733f22794222886bb0fa52c4b6ef0e52e98e Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Tue, 1 Aug 2017 10:52:14 -0600 Subject: [PATCH] DDPG Pendulum --- agent/DDPG_agent.py | 89 +++++++++++----------------- component/task.py | 24 ++------ main.py | 39 ++++++------ network/continuous_action_network.py | 8 ++- utils/config.py | 6 ++ 5 files changed, 69 insertions(+), 97 deletions(-) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 074e827..961947d 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -10,87 +10,67 @@ from utils import * import pickle class DDPGAgent: - def __init__(self, - task_fn, - actor_network_fn, - critic_network_fn, - actor_optimizer_fn, - critic_optimizer_fn, - replay_fn, - discount, - step_limit, - tau, - exploration_steps, - random_process_fn, - test_interval, - test_repetitions, - noise_decay_steps, - tag, - logger): - self.task = task_fn() - self.actor = actor_network_fn() - self.critic = critic_network_fn() - self.target_actor = actor_network_fn() - self.target_critic = critic_network_fn() + def __init__(self, config): + self.config = config + self.task = config.task_fn() + self.actor = config.actor_network_fn() + self.critic = config.critic_network_fn() + self.target_actor = config.actor_network_fn() + self.target_critic = config.critic_network_fn() self.target_actor.load_state_dict(self.actor.state_dict()) self.target_critic.load_state_dict(self.critic.state_dict()) - self.actor_opt = actor_optimizer_fn(self.actor.parameters()) - self.critic_opt = critic_optimizer_fn(self.critic.parameters()) - self.replay = replay_fn() - self.step_limit = step_limit - self.tau = tau - self.logger = logger - self.discount = discount - self.exploration_steps = exploration_steps - self.random_process = random_process_fn() + self.actor_opt = config.actor_optimizer_fn(self.actor.parameters()) + self.critic_opt = config.critic_optimizer_fn(self.critic.parameters()) + self.replay = config.replay_fn() + self.random_process = config.random_process_fn() self.criterion = nn.MSELoss() - self.test_interval = test_interval - self.test_repetitions = test_repetitions self.total_steps = 0 - self.tag = tag self.epsilon = 1.0 - self.d_epsilon = 1.0 / noise_decay_steps + self.d_epsilon = 1.0 / config.noise_decay_interval def soft_update(self, target, src): for target_param, param in zip(target.parameters(), src.parameters()): - target_param.data.copy_( - target_param.data * (1.0 - self.tau) + param.data * self.tau - ) + target_param.data.copy_(target_param.data * (1.0 - self.config.target_network_mix) + + param.data * self.config.target_network_mix) def episode(self, deterministic=False): self.random_process.reset_states() state = self.task.reset() + state = self.config.state_shift_fn(state) steps = 0 total_reward = 0.0 - while not self.step_limit or steps < self.step_limit: + while not self.config or steps < self.config.max_episode_length: action = self.actor.predict(np.stack([state])).flatten() - self.logger.histo_summary('action', action, self.total_steps) + self.config.logger.histo_summary('action', action, self.total_steps) if not deterministic: - if self.total_steps < self.exploration_steps: + if self.total_steps < self.config.exploration_steps: action = self.task.random_action() else: action += max(self.epsilon, 0) * self.random_process.sample() - self.logger.histo_summary('noised action', action, self.total_steps) + self.epsilon -= self.d_epsilon + self.config.logger.histo_summary('noised action', action, self.total_steps) next_state, reward, done, info = self.task.step(action) + next_state = self.config.state_shift_fn(next_state) + self.config.logger.scalar_summary('reward', reward, self.total_steps) + total_reward += reward + reward = self.config.reward_shift_fn(reward) if not deterministic: self.replay.feed([state, action, reward, next_state, int(done)]) self.total_steps += 1 - self.epsilon -= self.d_epsilon steps += 1 - total_reward += reward state = next_state if done: break - if not deterministic and self.total_steps > self.exploration_steps: + if not deterministic and self.total_steps > self.config.exploration_steps: experiences = self.replay.sample() states, actions, rewards, next_states, terminals = experiences q_next = self.target_critic.predict(next_states, self.target_actor.predict(next_states)) terminals = self.critic.to_torch_variable(terminals).unsqueeze(1) rewards = self.critic.to_torch_variable(rewards).unsqueeze(1) - q_next = self.discount * q_next * (1 - terminals) + q_next = self.config.discount * q_next * (1 - terminals) q_next.add_(rewards) q_next = Variable(q_next.data) q = self.critic.predict(states, actions) @@ -126,20 +106,21 @@ class DDPGAgent: reward = self.episode() rewards.append(reward) avg_reward = np.mean(rewards[-window_size:]) - self.logger.info('episode %d, reward %f, avg reward %f, total steps %d' % ( + self.config.logger.info('episode %d, reward %f, avg reward %f, total steps %d' % ( ep, reward, avg_reward, self.total_steps)) - if self.test_interval and ep % self.test_interval == 0: - self.logger.info('Testing...') - self.save('data/%sddpg-model-%s.bin' % (self.tag, self.task.name)) + if self.config.test_interval and ep % self.config.test_interval == 0: + self.config.logger.info('Testing...') + with open('data/%s-ddpg-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: + pickle.dump(self.actor.state_dict(), f) test_rewards = [] - for _ in range(self.test_repetitions): + for _ in range(self.config.test_repetitions): test_rewards.append(self.episode(True)) avg_reward = np.mean(test_rewards) avg_test_rewards.append(avg_reward) - self.logger.info('Avg reward %f(%f)' % ( - avg_reward, np.std(test_rewards) / np.sqrt(self.test_repetitions))) - with open('data/%sddpg-statistics-%s.bin' % (self.tag, self.task.name), 'wb') as f: + self.config.logger.info('Avg reward %f(%f)' % ( + avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions))) + with open('data/%s-ddpg-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: pickle.dump({'rewards': rewards, 'test_rewards': avg_test_rewards}, f) if avg_reward > self.task.success_threshold: diff --git a/component/task.py b/component/task.py index 6ef9bb5..b376cb8 100644 --- a/component/task.py +++ b/component/task.py @@ -80,22 +80,6 @@ class Pendulum(BasicTask): name = 'Pendulum-v0' success_threshold = -10 - def __init__(self): - BasicTask.__init__(self) - self.env = gym.make(self.name) - self.env._max_episode_steps = sys.maxsize - self.action_dim = self.env.action_space.shape[0] - self.state_dim = self.env.observation_space.shape[0] - - def step(self, action): - action = np.clip(action, -2, 2) - next_state, reward, done, info = self.env.step(action) - return next_state, reward, done, info - -class MountainCarContinuous(BasicTask): - name = 'MountainCarContinuous-v0' - success_threshold = 90 - def __init__(self): BasicTask.__init__(self) self.env = gym.make(self.name) @@ -104,15 +88,15 @@ class MountainCarContinuous(BasicTask): self.state_dim = self.env.observation_space.shape[0] def normalize_state(self, state): - state = (state - self.env.unwrapped.low_state) / \ - (self.env.unwrapped.high_state - self.env.unwrapped.low_state) + state = (state - self.env.observation_space.low) / \ + (self.env.observation_space.high - self.env.observation_space.low) state = state * 2 - 1 return state def step(self, action): - action = np.clip(action, -1, 1) + action = np.clip(action, -2, 2) next_state, reward, done, info = self.env.step(action) - return next_state, reward, done, info + return self.normalize_state(next_state), reward, done, info class BipedalWalker(BasicTask): name = 'BipedalWalker-v2' diff --git a/main.py b/main.py index a3001fa..e1dd212 100644 --- a/main.py +++ b/main.py @@ -68,7 +68,6 @@ def a3c_pendulum(): config = Config() config.task_fn = lambda: Pendulum() config.reward_shift_fn = lambda reward: reward / 10 - # config.task_fn = lambda: MountainCarContinuous() task = config.task_fn() config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001) config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) @@ -184,25 +183,25 @@ def a3c_pixel_atari(name): def ddpg_pendulum(): task_fn = lambda: Pendulum() task = task_fn() - config = dict() - config['task_fn'] = task_fn - config['actor_network_fn'] = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh) - config['critic_network_fn'] = lambda: DDPGCriticNet(task.state_dim, task.action_dim) - config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4) - config['critic_optimizer_fn'] =\ + config = Config() + config.task_fn = task_fn + config.actor_network_fn = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, 2) + config.critic_network_fn = lambda: DDPGCriticNet(task.state_dim, task.action_dim) + config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) + config.critic_optimizer_fn =\ lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) - config['replay_fn'] = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) - config['discount'] = 0.99 - config['step_limit'] = 200 - config['tau'] = 0.001 - config['exploration_steps'] = 100 - config['random_process_fn'] = \ + config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) + config.discount = 0.99 + config.max_episode_length = 200 + config.target_network_mix = 0.001 + config.exploration_steps = 100 + config.noise_decay_interval = 10000 + config.random_process_fn = \ lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) - config['test_interval'] = 10 - config['test_repetitions'] = 10 - config['tag'] = '' - config['logger'] = Logger('./log', gym.logger) - agent = DDPGAgent(**config) + config.test_interval = 10 + config.test_repetitions = 10 + config.logger = Logger('./log', gym.logger) + agent = DDPGAgent(config) agent.run() def ddpg_bipedal_walker(): @@ -237,8 +236,9 @@ if __name__ == '__main__': # dqn_cart_pole() # async_cart_pole() # a3c_cart_pole() - a3c_pendulum() + # a3c_pendulum() # a3c_walker() + ddpg_pendulum() # dqn_pixel_atari('PongNoFrameskip-v3') # async_pixel_atari('PongNoFrameskip-v3') @@ -248,5 +248,4 @@ if __name__ == '__main__': # async_pixel_atari('BreakoutNoFrameskip-v3') # a3c_pixel_atari('BreakoutNoFrameskip-v3') - # ddpg_pendulum() # ddpg_bipedal_walker() \ No newline at end of file diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index 27670c5..95e01ba 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -47,13 +47,15 @@ class DDPGActorNet(nn.Module, BasicNet): def __init__(self, state_dim, action_dim, - output_gate, + action_gate, + action_scale, gpu=False): super(DDPGActorNet, self).__init__() self.layer1 = nn.Linear(state_dim, 400) self.layer2 = nn.Linear(400, 300) self.layer3 = nn.Linear(300, action_dim) - self.output_gate = output_gate + self.action_gate = action_gate + self.action_scale = action_scale BasicNet.__init__(self, None, False, False) self.init_weights() @@ -76,7 +78,7 @@ class DDPGActorNet(nn.Module, BasicNet): x = F.relu(self.layer1(x)) x = F.relu(self.layer2(x)) x = self.layer3(x) - # x = self.output_gate(self.layer3(x)) + x = self.action_scale * self.action_gate(x) return x def predict(self, x, to_numpy=True): diff --git a/utils/config.py b/utils/config.py index 631bf8b..c9c88fc 100644 --- a/utils/config.py +++ b/utils/config.py @@ -8,10 +8,14 @@ class Config: def __init__(self): self.task_fn = None self.optimizer_fn = None + self.actor_optimizer_fn = None self.critic_optimizer_fn = None self.network_fn = None + self.actor_network_fn = None + self.critic_network_fn = None self.policy_fn = None self.replay_fn = None + self.random_process_fn = None self.discount = 0.99 self.target_network_update_freq = 0 self.max_episode_length = 0 @@ -28,6 +32,8 @@ class Config: self.gradient_clip = 40 self.entropy_weight = 0.01 self.gae_tau = 1.0 + self.noise_decay_interval = 0 + self.target_network_mix = 0.001 self.reward_shift_fn = lambda r: r self.state_shift_fn = lambda s: s self.action_shift_fn = lambda a: a