diff --git a/README.md b/README.md index b554876..9444ee9 100644 --- a/README.md +++ b/README.md @@ -49,14 +49,25 @@ variance unbounded, which is also included in the implementation. ## DDPG +![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DDPG-Pendulum-v0.png) + +DDPG is extremely unstable and is the most difficult algorithm to tune from my experience. And it cannot solve +Continuous Lunar Lander or Bipedal Walker. I never see a public DDPG implementation without a fixed random seed + that can solve tasks other than the family of Pendulum. If you find a bug or some successful practice, it will + be much appreciated to let me know that. + ## DPPO -The difference between my implementation and [DeepMind version](https://arxiv.org/abs/1707.02286) is: +![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DPPO.png) + +The difference between my implementation and [DeepMind's DPPO](https://arxiv.org/abs/1707.02286) is: 1. PPO stands for different algorithms. 2. I use a much simpler A3C-like synchronization protocol. The body of PPO is based on [this](https://github.com/alexis-jacq/Pytorch-DPPO), however that implementation has some critical bugs. + +I use 8 threads and a two tanh hidden layer network, each hidden layer has 64 hidden units. # Dependency * Open AI gym diff --git a/agent/A2C_agent.py b/agent/A2C_agent.py index aac5806..c8a2b52 100644 --- a/agent/A2C_agent.py +++ b/agent/A2C_agent.py @@ -27,10 +27,11 @@ class A2CAgent: state = self.task.reset() total_reward = 0.0 steps = 0 - while not self.config.max_episode_length or steps < self.config.max_episode_length: + while True: prob = self.learning_network.predict(np.stack([state]), True) action = self.policy.sample(prob, deterministic=deterministic) next_state, reward, done, info = self.task.step(action) + done = (done or (self.config.max_episode_length and steps > self.config.max_episode_length)) if not deterministic: self.replay.feed([state, action, reward, next_state, int(done)]) self.total_steps += 1 diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 7bf4ad2..1c5619b 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -98,38 +98,8 @@ class DDPGAgent: self.soft_update(self.target_network, self.learning_network) - return total_reward + return total_reward, steps def save(self, file_name): with open(file_name, 'wb') as f: - pickle.dump(self.actor.state_dict(), f) - - def run(self): - window_size = 100 - ep = 0 - rewards = [] - avg_test_rewards = [] - while True: - ep += 1 - reward = self.episode() - rewards.append(reward) - avg_reward = np.mean(rewards[-window_size:]) - self.config.logger.info('episode %d, reward %f, avg reward %f, total steps %d' % ( - ep, reward, avg_reward, self.total_steps)) - - if self.config.test_interval and ep % self.config.test_interval == 0: - self.config.logger.info('Testing...') - with open('data/%s-ddpg-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: - pickle.dump(self.actor.state_dict(), f) - test_rewards = [] - for _ in range(self.config.test_repetitions): - test_rewards.append(self.episode(True)) - avg_reward = np.mean(test_rewards) - avg_test_rewards.append(avg_reward) - self.config.logger.info('Avg reward %f(%f)' % ( - avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions))) - with open('data/%s-ddpg-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: - pickle.dump({'rewards': rewards, - 'test_rewards': avg_test_rewards}, f) - if avg_reward > self.task.success_threshold: - break + pickle.dump(self.learning_network.state_dict(), f) diff --git a/agent/DQN_agent.py b/agent/DQN_agent.py index a8d70f2..bcd9844 100644 --- a/agent/DQN_agent.py +++ b/agent/DQN_agent.py @@ -36,7 +36,7 @@ class DQNAgent: state = np.vstack(self.history_buffer) total_reward = 0.0 steps = 0 - while not self.config.max_episode_length or steps < self.config.max_episode_length: + while True: value = self.learning_network.predict(np.stack([self.task.normalize_state(state)]), False) value = value.cpu().data.numpy().flatten() if deterministic: @@ -46,6 +46,7 @@ class DQNAgent: else: action = self.policy.sample(value) next_state, reward, done, info = self.task.step(action) + done = (done or (self.config.max_episode_length and steps > self.config.max_episode_length)) self.history_buffer.pop(0) self.history_buffer.append(next_state) next_state = np.vstack(self.history_buffer) @@ -109,42 +110,6 @@ class DQNAgent: (steps, episode_time, episode_time / float(steps))) return total_reward, steps - def run(self): - window_size = 100 - ep = 0 - rewards = [] - steps = [] - avg_test_rewards = [] - while True: - ep += 1 - reward, step = self.episode() - steps.append(step) - rewards.append(reward) - avg_reward = np.mean(rewards[-window_size:]) - self.config.logger.info('episode %d, epsilon %f, reward %f, avg reward %f, total steps %d, episode step %d' % ( - ep, self.policy.epsilon, reward, avg_reward, self.total_steps, step)) - if self.config.episode_limit and ep > self.config.episode_limit: - return rewards, steps - - if ep % 100 == 0: - with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: - pickle.dump({'rewards': rewards, - 'steps': steps}, f) - - if self.config.test_interval and ep % self.config.test_interval == 0: - self.config.logger.info('Testing...') - with open('data/%s-dqn-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: - pickle.dump(self.learning_network.state_dict(), f) - test_rewards = [] - for _ in range(self.config.test_repetitions): - test_rewards.append(self.episode(True)) - avg_reward = np.mean(test_rewards) - avg_test_rewards.append(avg_reward) - self.config.logger.info('Avg reward %f(%f)' % ( - avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions))) - with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: - pickle.dump({'rewards': rewards, - 'steps': steps, - 'test_rewards': avg_test_rewards}, f) - if avg_reward > self.task.success_threshold: - break \ No newline at end of file + def save(self, file_name): + with open(file_name, 'wb') as f: + pickle.dump(self.learning_network.state_dict(), f) diff --git a/agent/MSDQN_agent.py b/agent/MSDQN_agent.py index 9faa416..7dfa0de 100644 --- a/agent/MSDQN_agent.py +++ b/agent/MSDQN_agent.py @@ -30,7 +30,7 @@ class MSDQNAgent: state = self.task.reset() total_reward = 0.0 steps = 0 - while not self.config.max_episode_length or steps < self.config.max_episode_length: + while True: value = self.learning_network.predict(np.stack([state]), True) value = value.cpu().data.numpy().flatten() if deterministic: @@ -40,6 +40,7 @@ class MSDQNAgent: else: action = self.policy.sample(value) next_state, reward, done, info = self.task.step(action) + done = (done or (self.config.max_episode_length and steps > self.config.max_episode_length)) if not deterministic: self.replay.feed([state, action, reward, next_state, int(done)]) self.total_steps += 1 @@ -98,43 +99,3 @@ class MSDQNAgent: self.config.logger.debug('episode steps %d, episode time %f, time per step %f' % (steps, episode_time, episode_time / float(steps))) return total_reward, steps - - def run(self): - window_size = 100 - ep = 0 - rewards = [] - steps = [] - avg_test_rewards = [] - while True: - ep += 1 - reward, step = self.episode() - steps.append(step) - rewards.append(reward) - avg_reward = np.mean(rewards[-window_size:]) - self.config.logger.info('episode %d, epsilon %f, reward %f, avg reward %f, total steps %d, episode step %d' % ( - ep, self.policy.epsilon, reward, avg_reward, self.total_steps, step)) - if self.config.episode_limit and ep > self.config.episode_limit: - return rewards, steps - - if ep % 100 == 0: - with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: - pickle.dump({'rewards': rewards, - 'steps': steps}, f) - - if self.config.test_interval and ep % self.config.test_interval == 0: - self.config.logger.info('Testing...') - with open('data/%s-dqn-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: - pickle.dump(self.learning_network.state_dict(), f) - test_rewards = [] - for _ in range(self.config.test_repetitions): - test_rewards.append(self.episode(True)) - avg_reward = np.mean(test_rewards) - avg_test_rewards.append(avg_reward) - self.config.logger.info('Avg reward %f(%f)' % ( - avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions))) - with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f: - pickle.dump({'rewards': rewards, - 'steps': steps, - 'test_rewards': avg_test_rewards}, f) - if avg_reward > self.task.success_threshold: - break \ No newline at end of file diff --git a/async_worker/actor_critic.py b/async_worker/actor_critic.py index 93ffa6f..423452c 100644 --- a/async_worker/actor_critic.py +++ b/async_worker/actor_critic.py @@ -24,11 +24,11 @@ class AdvantageActorCritic: steps = 0 total_reward = 0 pending = [] - while not config.stop_signal.value and \ - (not config.max_episode_length or steps < config.max_episode_length): + while not config.stop_signal.value: prob, log_prob, value = self.worker_network.predict(np.stack([state])) action = self.policy.sample(prob.data.numpy().flatten(), deterministic) next_state, reward, terminal, _ = self.task.step(action) + terminal = (terminal or (self.config.max_episode_length and steps > self.config.max_episode_length)) steps += 1 total_reward += reward diff --git a/async_worker/n_step_q.py b/async_worker/n_step_q.py index c0d30a8..b033b4d 100644 --- a/async_worker/n_step_q.py +++ b/async_worker/n_step_q.py @@ -25,11 +25,11 @@ class NStepQLearning: steps = 0 total_reward = 0 pending = [] - while not config.stop_signal.value and \ - (not config.max_episode_length or steps < config.max_episode_length): + while not config.stop_signal.value: q = self.worker_network.predict(np.stack([state])) action = self.policy.sample(q.data.numpy().flatten(), deterministic) next_state, reward, terminal, _ = self.task.step(action) + terminal = (terminal or (config.max_episode_length and steps >= config.max_episode_length)) steps += 1 total_reward += reward diff --git a/async_worker/one_step_q.py b/async_worker/one_step_q.py index fd41862..2e242ce 100644 --- a/async_worker/one_step_q.py +++ b/async_worker/one_step_q.py @@ -25,11 +25,11 @@ class OneStepQLearning: steps = 0 total_reward = 0 pending = [] - while not config.stop_signal.value and \ - (not config.max_episode_length or steps < config.max_episode_length): + while not config.stop_signal.value: q = self.worker_network.predict(np.stack([state])) action = self.policy.sample(q.data.numpy().flatten(), deterministic) next_state, reward, terminal, _ = self.task.step(action) + terminal = (terminal or (config.max_episode_length and steps >= config.max_episode_length)) steps += 1 total_reward += reward diff --git a/async_worker/one_step_sarsa.py b/async_worker/one_step_sarsa.py index f6d6464..b5123e9 100644 --- a/async_worker/one_step_sarsa.py +++ b/async_worker/one_step_sarsa.py @@ -27,9 +27,9 @@ class OneStepSarsa: steps = 0 total_reward = 0 pending = [] - while not config.stop_signal.value and \ - (not config.max_episode_length or steps < config.max_episode_length): + while not config.stop_signal.value: next_state, reward, terminal, _ = self.task.step(action) + terminal = (terminal or (config.max_episode_length and steps >= config.max_episode_length)) next_q = self.worker_network.predict(np.stack([next_state])) next_action = self.policy.sample(next_q.data.numpy().flatten(), deterministic) pending.append([q, action, reward, next_state, next_action]) diff --git a/images/DDPG-Pendulum-v0.png b/images/DDPG-Pendulum-v0.png new file mode 100644 index 0000000..4da5d1b Binary files /dev/null and b/images/DDPG-Pendulum-v0.png differ diff --git a/images/DPPO.png b/images/DPPO.png new file mode 100644 index 0000000..ec2a387 Binary files /dev/null and b/images/DPPO.png differ diff --git a/main.py b/main.py index 00e4d97..326f1a9 100644 --- a/main.py +++ b/main.py @@ -21,8 +21,7 @@ def dqn_cart_pole(): config.test_repetitions = 50 # config.double_q = True config.double_q = False - agent = DQNAgent(config) - agent.run() + run_episodes(DQNAgent(config)) def async_cart_pole(): config = Config() @@ -128,8 +127,7 @@ def dqn_pixel_atari(name): config.test_repetitions = 1 # config.double_q = True config.double_q = False - agent = DQNAgent(config) - agent.run() + run_episodes(DQNAgent(config)) def async_pixel_atari(name): config = Config() @@ -182,9 +180,9 @@ def ddpg_pendulum(): config = Config() config.task_fn = task_fn config.actor_network_fn = lambda: DeterministicActorNet( - task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.tanh) + task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.relu, batch_norm=False) config.critic_network_fn = lambda: DeterministicCriticNet( - task.state_dim, task.action_dim, non_linear=F.tanh) + task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False) config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) config.critic_optimizer_fn =\ @@ -199,17 +197,19 @@ def ddpg_pendulum(): lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) config.test_interval = 0 config.test_repetitions = 10 + config.save_interval = 50 config.logger = Logger('./log', gym.logger) - agent = DDPGAgent(config) - agent.run() + run_episodes(DDPGAgent(config)) def ddpg_lunar_lander(): task_fn = lambda: ContinuousLunarLander() task = task_fn() config = Config() config.task_fn = task_fn - config.actor_network_fn = lambda: DeterministicActorNet(task.state_dim, task.action_dim, F.tanh, 1) - config.critic_network_fn = lambda: DeterministicCriticNet(task.state_dim, task.action_dim) + config.actor_network_fn = lambda: DeterministicActorNet( + task.state_dim, task.action_dim, F.tanh, 1, batch_norm=True) + config.critic_network_fn = lambda: DeterministicCriticNet( + task.state_dim, task.action_dim, batch_norm=True) config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) config.critic_optimizer_fn =\ @@ -224,9 +224,9 @@ def ddpg_lunar_lander(): lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) config.test_interval = 0 config.test_repetitions = 10 + config.save_interval = 50 config.logger = Logger('./log', gym.logger) - agent = DDPGAgent(config) - agent.run() + run_episodes(DDPGAgent(config)) def ddpg_walker(): task_fn = lambda: BipedalWalker() @@ -252,9 +252,9 @@ def ddpg_walker(): lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) config.test_interval = 0 config.test_repetitions = 5 + config.save_interval = 50 config.logger = Logger('./log', gym.logger) - agent = DDPGAgent(config) - agent.run() + run_episodes(DDPGAgent(config)) def dqn_fruit(): config = Config() @@ -275,10 +275,8 @@ def dqn_fruit(): config.test_interval = 0 config.test_repetitions = 10 config.episode_limit = 5000 - config.tag = 'vanilla-%f' % (0.001) config.double_q = False - agent = DQNAgent(config) - agent.run() + run_episodes(DQNAgent(config)) def hrdqn_fruit(): config = Config() @@ -302,8 +300,7 @@ def hrdqn_fruit(): # config.target_type = config.q_target config.double_q = False config.episode_limit = 5000 - agent = DQNAgent(config) - agent.run() + run_episodes(DQNAgent(config)) def hrmsdqn_fruit(): config = Config() @@ -328,13 +325,11 @@ def hrmsdqn_fruit(): # config.target_type = config.q_target config.double_q = False config.episode_limit = 5000 - agent = MSDQNAgent(config) - agent.run() + run_episodes(MSDQNAgent(config)) def ppo_pendulum(): config = Config() config.task_fn = lambda: Pendulum() - # config.task_fn = lambda: BipedalWalker() task = config.task_fn() config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim) config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim) @@ -351,7 +346,34 @@ def ppo_pendulum(): config.test_interval = 1 config.test_repetitions = 1 config.max_episode_length = 200 - # config.max_episode_length = 999 + config.entropy_weight = 0 + config.gradient_clip = 40 + config.rollout_length = 10000 + config.optimize_epochs = 1 + config.ppo_ratio_clip = 0.2 + config.logger = Logger('./log', gym.logger) + agent = AsyncAgent(config) + agent.run() + +def ppo_walker(): + config = Config() + config.task_fn = lambda: BipedalWalker() + task = config.task_fn() + config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim) + config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim) + config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) + config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) + config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) + + config.policy_fn = lambda: GaussianPolicy() + config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=2048) + config.worker = ProximalPolicyOptimization + config.discount = 0.99 + config.gae_tau = 0.97 + config.num_workers = 8 + config.test_interval = 1 + config.test_repetitions = 1 + config.max_episode_length = 999 config.entropy_weight = 0 config.gradient_clip = 40 config.rollout_length = 10000 @@ -362,18 +384,19 @@ def ppo_pendulum(): agent.run() if __name__ == '__main__': - gym.logger.setLevel(logging.DEBUG) - # gym.logger.setLevel(logging.INFO) + # gym.logger.setLevel(logging.DEBUG) + gym.logger.setLevel(logging.INFO) - # dqn_cart_pole() + dqn_cart_pole() # async_cart_pole() # a3c_cart_pole() # a3c_pendulum() # a3c_walker() # ddpg_pendulum() - ddpg_lunar_lander() + # ddpg_lunar_lander() # ddpg_walker() # ppo_pendulum() + # ppo_walker() # dqn_fruit() # hrdqn_fruit() diff --git a/utils/__init__.py b/utils/__init__.py index 63ad9fd..69a6be6 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -1,5 +1,6 @@ from config import * from normalizer import * +from run import * try: from tf_logger import Logger except: diff --git a/utils/config.py b/utils/config.py index 32374a2..cb33f8c 100644 --- a/utils/config.py +++ b/utils/config.py @@ -46,3 +46,4 @@ class Config: self.master_optimizer_fn = None self.num_heads = 10 self.min_epsilon = 0 + self.save_interval = 0 diff --git a/utils/run.py b/utils/run.py new file mode 100644 index 0000000..b277163 --- /dev/null +++ b/utils/run.py @@ -0,0 +1,53 @@ +####################################################################### +# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) # +# Permission given to modify the code as long as you keep this # +# declaration at the top # +####################################################################### + +import numpy as np +import pickle + + +def run_episodes(agent): + config = agent.config + window_size = 100 + ep = 0 + rewards = [] + steps = [] + avg_test_rewards = [] + agent_type = agent.__class__.__name__ + while True: + ep += 1 + reward, step = agent.episode() + rewards.append(reward) + steps.append(step) + avg_reward = np.mean(rewards[-window_size:]) + config.logger.info('episode %d, reward %f, avg reward %f, total steps %d, episode step %d' % ( + ep, reward, avg_reward, agent.total_steps, step)) + + if config.save_interval and ep % config.save_interval == 0: + with open('data/%s-%s-online-stats-%s.bin' % ( + agent_type, config.tag, agent.task.name), 'wb') as f: + pickle.dump([steps, rewards], f) + + if config.episode_limit and ep > config.episode_limit: + break + + if config.test_interval and ep % config.test_interval == 0: + config.logger.info('Testing...') + agent.save('data/%s-%s-model-%s.bin' % (agent_type, config.tag, agent.task.name)) + test_rewards = [] + for _ in range(config.test_repetitions): + test_rewards.append(agent.episode(True)) + avg_reward = np.mean(test_rewards) + avg_test_rewards.append(avg_reward) + config.logger.info('Avg reward %f(%f)' % ( + avg_reward, np.std(test_rewards) / np.sqrt(config.test_repetitions))) + with open('data/%s-%s-all-stats-%s.bin' % (agent_type, config.tag, agent.task.name), 'wb') as f: + pickle.dump({'rewards': rewards, + 'steps': steps, + 'test_rewards': avg_test_rewards}, f) + if avg_reward > agent.task.success_threshold: + break + + return steps, rewards, avg_test_rewards \ No newline at end of file