Major update

README.md

@@ -49,14 +49,25 @@ variance unbounded, which is also included in the implementation.
 
 ## DDPG
 
+![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DDPG-Pendulum-v0.png)
+
+DDPG is extremely unstable and is the most difficult algorithm to tune from my experience. And it cannot solve 
+Continuous Lunar Lander or Bipedal Walker. I never see a public DDPG implementation without a fixed random seed
+ that can solve tasks other than the family of Pendulum. If you find a bug or some successful practice, it will 
+ be much appreciated to let me know that.
+
 ## DPPO
 
-The difference between my implementation and [DeepMind version](https://arxiv.org/abs/1707.02286) is:
+![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DPPO.png)
+
+The difference between my implementation and [DeepMind's DPPO](https://arxiv.org/abs/1707.02286) is:
 1. PPO stands for different algorithms.
 2. I use a much simpler A3C-like synchronization protocol. 
 
 The body of PPO is based on [this](https://github.com/alexis-jacq/Pytorch-DPPO), however that implementation has some
  critical bugs. 
+ 
+I use 8 threads and a two tanh hidden layer network, each hidden layer has 64 hidden units.
 
 # Dependency
 * Open AI gym

agent/A2C_agent.py

@@ -27,10 +27,11 @@ class A2CAgent:
         state = self.task.reset()
         total_reward = 0.0
         steps = 0
-        while not self.config.max_episode_length or steps < self.config.max_episode_length:
+        while True:
             prob = self.learning_network.predict(np.stack([state]), True)
             action = self.policy.sample(prob, deterministic=deterministic)
             next_state, reward, done, info = self.task.step(action)
+            done = (done or (self.config.max_episode_length and steps > self.config.max_episode_length))
             if not deterministic:
                 self.replay.feed([state, action, reward, next_state, int(done)])
                 self.total_steps += 1

agent/DDPG_agent.py

@@ -98,38 +98,8 @@ class DDPGAgent:
 
                 self.soft_update(self.target_network, self.learning_network)
 
-        return total_reward
+        return total_reward, steps
 
     def save(self, file_name):
         with open(file_name, 'wb') as f:
-            pickle.dump(self.actor.state_dict(), f)
-
-    def run(self):
-        window_size = 100
-        ep = 0
-        rewards = []
-        avg_test_rewards = []
-        while True:
-            ep += 1
-            reward = self.episode()
-            rewards.append(reward)
-            avg_reward = np.mean(rewards[-window_size:])
-            self.config.logger.info('episode %d, reward %f, avg reward %f, total steps %d' % (
-                ep, reward, avg_reward, self.total_steps))
-
-            if self.config.test_interval and ep % self.config.test_interval == 0:
-                self.config.logger.info('Testing...')
-                with open('data/%s-ddpg-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
-                    pickle.dump(self.actor.state_dict(), f)
-                test_rewards = []
-                for _ in range(self.config.test_repetitions):
-                    test_rewards.append(self.episode(True))
-                avg_reward = np.mean(test_rewards)
-                avg_test_rewards.append(avg_reward)
-                self.config.logger.info('Avg reward %f(%f)' % (
-                    avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions)))
-                with open('data/%s-ddpg-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
-                    pickle.dump({'rewards': rewards,
-                                 'test_rewards': avg_test_rewards}, f)
-                if avg_reward > self.task.success_threshold:
-                    break
+            pickle.dump(self.learning_network.state_dict(), f)

agent/DQN_agent.py

@@ -36,7 +36,7 @@ class DQNAgent:
         state = np.vstack(self.history_buffer)
         total_reward = 0.0
         steps = 0
-        while not self.config.max_episode_length or steps < self.config.max_episode_length:
+        while True:
             value = self.learning_network.predict(np.stack([self.task.normalize_state(state)]), False)
             value = value.cpu().data.numpy().flatten()
             if deterministic:
@@ -46,6 +46,7 @@ class DQNAgent:
             else:
                 action = self.policy.sample(value)
             next_state, reward, done, info = self.task.step(action)
+            done = (done or (self.config.max_episode_length and steps > self.config.max_episode_length))
             self.history_buffer.pop(0)
             self.history_buffer.append(next_state)
             next_state = np.vstack(self.history_buffer)
@@ -109,42 +110,6 @@ class DQNAgent:
                           (steps, episode_time, episode_time / float(steps)))
         return total_reward, steps
 
-    def run(self):
-        window_size = 100
-        ep = 0
-        rewards = []
-        steps = []
-        avg_test_rewards = []
-        while True:
-            ep += 1
-            reward, step = self.episode()
-            steps.append(step)
-            rewards.append(reward)
-            avg_reward = np.mean(rewards[-window_size:])
-            self.config.logger.info('episode %d, epsilon %f, reward %f, avg reward %f, total steps %d, episode step %d' % (
-                ep, self.policy.epsilon, reward, avg_reward, self.total_steps, step))
-            if self.config.episode_limit and ep > self.config.episode_limit:
-                return rewards, steps
-
-            if ep % 100 == 0:
-                with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
-                    pickle.dump({'rewards': rewards,
-                                 'steps': steps}, f)
-
-            if self.config.test_interval and ep % self.config.test_interval == 0:
-                self.config.logger.info('Testing...')
-                with open('data/%s-dqn-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
-                    pickle.dump(self.learning_network.state_dict(), f)
-                test_rewards = []
-                for _ in range(self.config.test_repetitions):
-                    test_rewards.append(self.episode(True))
-                avg_reward = np.mean(test_rewards)
-                avg_test_rewards.append(avg_reward)
-                self.config.logger.info('Avg reward %f(%f)' % (
-                    avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions)))
-                with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
-                    pickle.dump({'rewards': rewards,
-                                 'steps': steps,
-                                 'test_rewards': avg_test_rewards}, f)
-                if avg_reward > self.task.success_threshold:
-                    break
+    def save(self, file_name):
+        with open(file_name, 'wb') as f:
+            pickle.dump(self.learning_network.state_dict(), f)

agent/MSDQN_agent.py

@@ -30,7 +30,7 @@ class MSDQNAgent:
         state = self.task.reset()
         total_reward = 0.0
         steps = 0
-        while not self.config.max_episode_length or steps < self.config.max_episode_length:
+        while True:
             value = self.learning_network.predict(np.stack([state]), True)
             value = value.cpu().data.numpy().flatten()
             if deterministic:
@@ -40,6 +40,7 @@ class MSDQNAgent:
             else:
                 action = self.policy.sample(value)
             next_state, reward, done, info = self.task.step(action)
+            done = (done or (self.config.max_episode_length and steps > self.config.max_episode_length))
             if not deterministic:
                 self.replay.feed([state, action, reward, next_state, int(done)])
                 self.total_steps += 1
@@ -98,43 +99,3 @@ class MSDQNAgent:
         self.config.logger.debug('episode steps %d, episode time %f, time per step %f' %
                           (steps, episode_time, episode_time / float(steps)))
         return total_reward, steps
-
-    def run(self):
-        window_size = 100
-        ep = 0
-        rewards = []
-        steps = []
-        avg_test_rewards = []
-        while True:
-            ep += 1
-            reward, step = self.episode()
-            steps.append(step)
-            rewards.append(reward)
-            avg_reward = np.mean(rewards[-window_size:])
-            self.config.logger.info('episode %d, epsilon %f, reward %f, avg reward %f, total steps %d, episode step %d' % (
-                ep, self.policy.epsilon, reward, avg_reward, self.total_steps, step))
-            if self.config.episode_limit and ep > self.config.episode_limit:
-                return rewards, steps
-
-            if ep % 100 == 0:
-                with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
-                    pickle.dump({'rewards': rewards,
-                                 'steps': steps}, f)
-
-            if self.config.test_interval and ep % self.config.test_interval == 0:
-                self.config.logger.info('Testing...')
-                with open('data/%s-dqn-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
-                    pickle.dump(self.learning_network.state_dict(), f)
-                test_rewards = []
-                for _ in range(self.config.test_repetitions):
-                    test_rewards.append(self.episode(True))
-                avg_reward = np.mean(test_rewards)
-                avg_test_rewards.append(avg_reward)
-                self.config.logger.info('Avg reward %f(%f)' % (
-                    avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions)))
-                with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
-                    pickle.dump({'rewards': rewards,
-                                 'steps': steps,
-                                 'test_rewards': avg_test_rewards}, f)
-                if avg_reward > self.task.success_threshold:
-                    break

async_worker/actor_critic.py

@@ -24,11 +24,11 @@ class AdvantageActorCritic:
         steps = 0
         total_reward = 0
         pending = []
-        while not config.stop_signal.value and \
-                (not config.max_episode_length or steps < config.max_episode_length):
+        while not config.stop_signal.value:
             prob, log_prob, value = self.worker_network.predict(np.stack([state]))
             action = self.policy.sample(prob.data.numpy().flatten(), deterministic)
             next_state, reward, terminal, _ = self.task.step(action)
+            terminal = (terminal or (self.config.max_episode_length and steps > self.config.max_episode_length))
 
             steps += 1
             total_reward += reward

async_worker/n_step_q.py

@@ -25,11 +25,11 @@ class NStepQLearning:
         steps = 0
         total_reward = 0
         pending = []
-        while not config.stop_signal.value and \
-                (not config.max_episode_length or steps < config.max_episode_length):
+        while not config.stop_signal.value:
             q = self.worker_network.predict(np.stack([state]))
             action = self.policy.sample(q.data.numpy().flatten(), deterministic)
             next_state, reward, terminal, _ = self.task.step(action)
+            terminal = (terminal or (config.max_episode_length and steps >= config.max_episode_length))
 
             steps += 1
             total_reward += reward

async_worker/one_step_q.py

@@ -25,11 +25,11 @@ class OneStepQLearning:
         steps = 0
         total_reward = 0
         pending = []
-        while not config.stop_signal.value and \
-                (not config.max_episode_length or steps < config.max_episode_length):
+        while not config.stop_signal.value:
             q = self.worker_network.predict(np.stack([state]))
             action = self.policy.sample(q.data.numpy().flatten(), deterministic)
             next_state, reward, terminal, _ = self.task.step(action)
+            terminal = (terminal or (config.max_episode_length and steps >= config.max_episode_length))
 
             steps += 1
             total_reward += reward

async_worker/one_step_sarsa.py

@@ -27,9 +27,9 @@ class OneStepSarsa:
         steps = 0
         total_reward = 0
         pending = []
-        while not config.stop_signal.value and \
-                (not config.max_episode_length or steps < config.max_episode_length):
+        while not config.stop_signal.value:
             next_state, reward, terminal, _ = self.task.step(action)
+            terminal = (terminal or (config.max_episode_length and steps >= config.max_episode_length))
             next_q = self.worker_network.predict(np.stack([next_state]))
             next_action = self.policy.sample(next_q.data.numpy().flatten(), deterministic)
             pending.append([q, action, reward, next_state, next_action])

main.py

@@ -21,8 +21,7 @@ def dqn_cart_pole():
     config.test_repetitions = 50
     # config.double_q = True
     config.double_q = False
-    agent = DQNAgent(config)
-    agent.run()
+    run_episodes(DQNAgent(config))
 
 def async_cart_pole():
     config = Config()
@@ -128,8 +127,7 @@ def dqn_pixel_atari(name):
     config.test_repetitions = 1
     # config.double_q = True
     config.double_q = False
-    agent = DQNAgent(config)
-    agent.run()
+    run_episodes(DQNAgent(config))
 
 def async_pixel_atari(name):
     config = Config()
@@ -182,9 +180,9 @@ def ddpg_pendulum():
     config = Config()
     config.task_fn = task_fn
     config.actor_network_fn = lambda: DeterministicActorNet(
-        task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.tanh)
+        task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.relu, batch_norm=False)
     config.critic_network_fn = lambda: DeterministicCriticNet(
-        task.state_dim, task.action_dim, non_linear=F.tanh)
+        task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False)
     config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
     config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
     config.critic_optimizer_fn =\
@@ -199,17 +197,19 @@ def ddpg_pendulum():
         lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
     config.test_interval = 0
     config.test_repetitions = 10
+    config.save_interval = 50
     config.logger = Logger('./log', gym.logger)
-    agent = DDPGAgent(config)
-    agent.run()
+    run_episodes(DDPGAgent(config))
 
 def ddpg_lunar_lander():
     task_fn = lambda: ContinuousLunarLander()
     task = task_fn()
     config = Config()
     config.task_fn = task_fn
-    config.actor_network_fn = lambda: DeterministicActorNet(task.state_dim, task.action_dim, F.tanh, 1)
-    config.critic_network_fn = lambda: DeterministicCriticNet(task.state_dim, task.action_dim)
+    config.actor_network_fn = lambda: DeterministicActorNet(
+        task.state_dim, task.action_dim, F.tanh, 1, batch_norm=True)
+    config.critic_network_fn = lambda: DeterministicCriticNet(
+        task.state_dim, task.action_dim, batch_norm=True)
     config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
     config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
     config.critic_optimizer_fn =\
@@ -224,9 +224,9 @@ def ddpg_lunar_lander():
         lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
     config.test_interval = 0
     config.test_repetitions = 10
+    config.save_interval = 50
     config.logger = Logger('./log', gym.logger)
-    agent = DDPGAgent(config)
-    agent.run()
+    run_episodes(DDPGAgent(config))
 
 def ddpg_walker():
     task_fn = lambda: BipedalWalker()
@@ -252,9 +252,9 @@ def ddpg_walker():
         lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
     config.test_interval = 0
     config.test_repetitions = 5
+    config.save_interval = 50
     config.logger = Logger('./log', gym.logger)
-    agent = DDPGAgent(config)
-    agent.run()
+    run_episodes(DDPGAgent(config))
 
 def dqn_fruit():
     config = Config()
@@ -275,10 +275,8 @@ def dqn_fruit():
     config.test_interval = 0
     config.test_repetitions = 10
     config.episode_limit = 5000
-    config.tag = 'vanilla-%f' % (0.001)
     config.double_q = False
-    agent = DQNAgent(config)
-    agent.run()
+    run_episodes(DQNAgent(config))
 
 def hrdqn_fruit():
     config = Config()
@@ -302,8 +300,7 @@ def hrdqn_fruit():
     # config.target_type = config.q_target
     config.double_q = False
     config.episode_limit = 5000
-    agent = DQNAgent(config)
-    agent.run()
+    run_episodes(DQNAgent(config))
 
 def hrmsdqn_fruit():
     config = Config()
@@ -328,13 +325,11 @@ def hrmsdqn_fruit():
     # config.target_type = config.q_target
     config.double_q = False
     config.episode_limit = 5000
-    agent = MSDQNAgent(config)
-    agent.run()
+    run_episodes(MSDQNAgent(config))
 
 def ppo_pendulum():
     config = Config()
     config.task_fn = lambda: Pendulum()
-    # config.task_fn = lambda: BipedalWalker()
     task = config.task_fn()
     config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim)
     config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim)
@@ -351,7 +346,34 @@ def ppo_pendulum():
     config.test_interval = 1
     config.test_repetitions = 1
     config.max_episode_length = 200
-    # config.max_episode_length = 999
+    config.entropy_weight = 0
+    config.gradient_clip = 40
+    config.rollout_length = 10000
+    config.optimize_epochs = 1
+    config.ppo_ratio_clip = 0.2
+    config.logger = Logger('./log', gym.logger)
+    agent = AsyncAgent(config)
+    agent.run()
+
+def ppo_walker():
+    config = Config()
+    config.task_fn = lambda: BipedalWalker()
+    task = config.task_fn()
+    config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim)
+    config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim)
+    config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
+    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
+    config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
+
+    config.policy_fn = lambda: GaussianPolicy()
+    config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=2048)
+    config.worker = ProximalPolicyOptimization
+    config.discount = 0.99
+    config.gae_tau = 0.97
+    config.num_workers = 8
+    config.test_interval = 1
+    config.test_repetitions = 1
+    config.max_episode_length = 999
     config.entropy_weight = 0
     config.gradient_clip = 40
     config.rollout_length = 10000
@@ -362,18 +384,19 @@ def ppo_pendulum():
     agent.run()
 
 if __name__ == '__main__':
-    gym.logger.setLevel(logging.DEBUG)
-    # gym.logger.setLevel(logging.INFO)
+    # gym.logger.setLevel(logging.DEBUG)
+    gym.logger.setLevel(logging.INFO)
 
-    # dqn_cart_pole()
+    dqn_cart_pole()
     # async_cart_pole()
     # a3c_cart_pole()
     # a3c_pendulum()
     # a3c_walker()
     # ddpg_pendulum()
-    ddpg_lunar_lander()
+    # ddpg_lunar_lander()
     # ddpg_walker()
     # ppo_pendulum()
+    # ppo_walker()
 
     # dqn_fruit()
     # hrdqn_fruit()

utils/__init__.py

@@ -1,5 +1,6 @@
 from config import *
 from normalizer import *
+from run import *
 try:
     from tf_logger import Logger
 except:

utils/config.py

@@ -46,3 +46,4 @@ class Config:
         self.master_optimizer_fn = None
         self.num_heads = 10
         self.min_epsilon = 0
+        self.save_interval = 0

utils/run.py

@@ -0,0 +1,53 @@
+#######################################################################
+# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
+# Permission given to modify the code as long as you keep this        #
+# declaration at the top                                              #
+#######################################################################
+
+import numpy as np
+import pickle
+
+
+def run_episodes(agent):
+    config = agent.config
+    window_size = 100
+    ep = 0
+    rewards = []
+    steps = []
+    avg_test_rewards = []
+    agent_type = agent.__class__.__name__
+    while True:
+        ep += 1
+        reward, step = agent.episode()
+        rewards.append(reward)
+        steps.append(step)
+        avg_reward = np.mean(rewards[-window_size:])
+        config.logger.info('episode %d, reward %f, avg reward %f, total steps %d, episode step %d' % (
+            ep, reward, avg_reward, agent.total_steps, step))
+
+        if config.save_interval and ep % config.save_interval == 0:
+            with open('data/%s-%s-online-stats-%s.bin' % (
+                    agent_type, config.tag, agent.task.name), 'wb') as f:
+                pickle.dump([steps, rewards], f)
+
+        if config.episode_limit and ep > config.episode_limit:
+            break
+
+        if config.test_interval and ep % config.test_interval == 0:
+            config.logger.info('Testing...')
+            agent.save('data/%s-%s-model-%s.bin' % (agent_type, config.tag, agent.task.name))
+            test_rewards = []
+            for _ in range(config.test_repetitions):
+                test_rewards.append(agent.episode(True))
+            avg_reward = np.mean(test_rewards)
+            avg_test_rewards.append(avg_reward)
+            config.logger.info('Avg reward %f(%f)' % (
+                avg_reward, np.std(test_rewards) / np.sqrt(config.test_repetitions)))
+            with open('data/%s-%s-all-stats-%s.bin' % (agent_type, config.tag, agent.task.name), 'wb') as f:
+                pickle.dump({'rewards': rewards,
+                             'steps': steps,
+                             'test_rewards': avg_test_rewards}, f)
+            if avg_reward > agent.task.success_threshold:
+                break
+
+    return steps, rewards, avg_test_rewards