Update Readme

README.md

@@ -1 +1,8 @@
-# DeepRL by PyTorch
+#DeepRL
+> Highly modularized implementation of popular deep RL algorithms powered by PyTorch
+* Deep Q-Learning
+* Asynchronous One-Step Q-Learning
+
+>Benchmarked by classical control tasks (CartPole, LunarLander). Atari games will make it difficult to replicate in a regular laptop without a good GPU. However it's fairly easy to adapt the components to fit Atari games.
+
+>Try it out from ```main.py```!

async_agent.py

@@ -13,7 +13,7 @@ from network import *
 
 class AsyncAgent:
     def __init__(self, task_fn, network_fn, optimizer_fn, policy_fn, discount, step_limit,
-                 target_network_update_freq, n_workers, batch_size, test_interval):
+                 target_network_update_freq, n_workers, batch_size, test_interval, test_repeats):
         self.network_fn = network_fn
         self.learning_network = network_fn()
         self.learning_network.share_memory()
@@ -35,14 +35,14 @@ class AsyncAgent:
         self.n_workers = n_workers
         self.batch_size = batch_size
         self.test_interval = test_interval
+        self.test_repeats = test_repeats
 
-    def deterministic_episode(self, task):
+    def deterministic_episode(self, task, network):
         state = np.asarray([task.reset()])
         total_rewards = 0
         steps = 0
         while True and steps < self.step_limit:
-            with self.network_lock:
-                action_values = self.learning_network.predict(state)
+            action_values = network.predict(state)
             steps += 1
             action = np.argmax(action_values.flatten())
             state, reward, terminal, _ = task.step(action)
@@ -68,10 +68,14 @@ class AsyncAgent:
         terminal = True
         episode = 0
         episode_steps = 0
+        episode_return = 0
         while True and not self.stop_signal.value:
             batch_states, batch_actions, batch_rewards = [], [], []
             if terminal:
+                if id == 0:
+                    print 'worker %d, episode %d, return %f' % (id, episode, episode_return)
                 episode_steps = 0
+                episode_return = 0
                 episode += 1
                 policy.update_epsilon()
                 terminal = False
@@ -86,6 +90,7 @@ class AsyncAgent:
                 action = policy.sample(value.flatten())
                 batch_actions.append(action)
                 state, reward, terminal, _ = task.step(action)
+                episode_return += reward
                 state = state.reshape([1, -1])
                 if not terminal:
                     with self.network_lock:
@@ -93,6 +98,9 @@ class AsyncAgent:
                     reward += self.discount * q_next
                 batch_rewards.append(reward)
 
+            if episode_steps > self.step_limit:
+                terminal = True
+
             worker_network.zero_grad()
             worker_network.gradient(np.vstack(batch_states), batch_actions, batch_rewards)
             self.async_update(worker_network, optimizer)
@@ -106,13 +114,15 @@ class AsyncAgent:
         procs = [mp.Process(target=self.worker, args=(i, )) for i in range(self.n_workers)]
         for p in procs: p.start()
         task = self.task_fn()
+        test_network = self.network_fn()
         while True:
             if self.total_steps.value % self.test_interval == 0:
-                test_repeats = 5
-                rewards = np.zeros(test_repeats)
-                for i in range(test_repeats):
-                    rewards[i] = self.deterministic_episode(task)
-                print 'total stpes: %d, test process epsidoe reward: %f' %\
+                with self.network_lock:
+                    test_network.load_state_dict(self.learning_network.state_dict())
+                rewards = np.zeros(self.test_repeats)
+                for i in range(self.test_repeats):
+                    rewards[i] = self.deterministic_episode(task, test_network)
+                print 'total steps: %d, averaged return per episode: %f' %\
                       (self.total_steps.value, np.mean(rewards))
                 if np.mean(rewards) > task.success_threshold:
                     self.stop_signal.value = True

dqn_agent.py

@@ -62,7 +62,8 @@ class DQNAgent:
             rewards.append(reward)
             if len(rewards) > window_size:
                 reward = np.mean(rewards[-window_size:])
-            print 'episode %d: %f' % (ep, reward)
+            print 'episode %d, epsilon %f, reward %f' % (
+                ep, self.policy.epsilon, reward)
             if reward > self.task.success_threshold:
                 break
 

main.py

@@ -13,6 +13,37 @@ def async_cart_pole():
     config['n_workers'] = 8
     config['batch_size'] = 5
     config['test_interval'] = 500
+    config['test_repeats'] = 5
+    agent = AsyncAgent(**config)
+    agent.run()
+
+# Mountain Car is fairly unstable
+def dqn_mountain_car():
+    config = dict()
+    config['task_fn'] = lambda: MountainCar()
+    config['optimizer_fn'] = lambda params: torch.optim.SGD(params, 0.001)
+    config['network_fn'] = lambda optimizer_fn: FullyConnectedNet([2, 50, 200, 3], optimizer_fn)
+    config['policy_fn'] = lambda: GreedyPolicy(epsilon=0.5, end_episode=500, min_epsilon=0.1)
+    config['replay_fn'] = lambda: Replay(memory_size=10000, batch_size=10)
+    config['discount'] = 0.99
+    config['target_network_update_freq'] = 1000
+    config['step_limit'] = 5000
+    agent = DQNAgent(**config)
+    agent.run()
+
+def async_lunar_lander():
+    config = dict()
+    config['task_fn'] = lambda: LunarLander()
+    config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001)
+    config['network_fn'] = lambda: FullyConnectedNet([8, 50, 200, 4])
+    config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, end_episode=2000, min_epsilon=0.05)
+    config['discount'] = 0.99
+    config['target_network_update_freq'] = 200
+    config['step_limit'] = 5000
+    config['n_workers'] = 8
+    config['batch_size'] = 10
+    config['test_interval'] = 1000
+    config['test_repeats'] = 5
     agent = AsyncAgent(**config)
     agent.run()
 
@@ -31,4 +62,6 @@ def dqn_cart_pole():
 
 if __name__ == '__main__':
     async_cart_pole()
+    # async_lunar_lander()
     # dqn_cart_pole()
+    # dqn_mountain_car()

task.py

@@ -3,11 +3,8 @@
 # Permission given to modify the code as long as you keep this        #
 # declaration at the top                                              #
 #######################################################################
-
 import gym
 import sys
-from dqn_agent import *
-import torch.optim
 
 class BasicTask:
     def transfer_state(self, state):
@@ -22,21 +19,12 @@ class BasicTask:
         return next_state, reward, done, info
 
 class MountainCar(BasicTask):
-    state_space_size = 2
-    action_space_size = 3
     name = 'MountainCar-v0'
     success_threshold = -110
-    discount = 0.99
-    step_limit = 5000
-    target_network_update_freq = 1000
 
     def __init__(self):
         self.env = gym.make(self.name)
         self.env._max_episode_steps = sys.maxsize
-        self.optimizer_fn = lambda params: torch.optim.SGD(params, 0.001)
-        self.network_fn = lambda optimizer_fn: FullyConnectedNet([self.state_space_size, 50, 200, self.action_space_size], optimizer_fn)
-        self.policy_fn = lambda: GreedyPolicy(epsilon=0.5, decay_factor=0.95, min_epsilon=0.1)
-        self.replay_fn = lambda: Replay(memory_size=10000, batch_size=10)
 
 class CartPole(BasicTask):
     name = 'CartPole-v0'
@@ -44,3 +32,10 @@ class CartPole(BasicTask):
 
     def __init__(self):
         self.env = gym.make(self.name)
+
+class LunarLander(BasicTask):
+    name = 'LunarLander-v2'
+    success_threshold = 200
+
+    def __init__(self):
+        self.env = gym.make(self.name)