A3C

README.md

@@ -4,6 +4,7 @@
 * Asynchronous One-Step Q-Learning
 * Asynchronous One-Step Sarsa 
 * Asynchronous N-Step Q-Learning
+* Asynchronous Advantage Actor Critic (A3C)
 
 >Benchmarked by classical control tasks (CartPole, LunarLander). Atari games will make it difficult to replicate in a regular laptop without a good GPU. However it's fairly easy to adapt the components to fit Atari games.
 

bootstrap.py

@@ -43,3 +43,16 @@ def OneStepSarsa(batch_states, batch_actions, batch_rewards,
     batch_actions.pop(-1)
     batch_rewards = np.asarray(batch_rewards) + agent.discount * q_next
     return batch_rewards
+
+def AdvantageActorCritic(batch_states, batch_actions, batch_rewards,
+                         tailing_state, tailing_action, terminal, agent):
+    if terminal:
+        reward = 0
+    else:
+        with agent.network_lock:
+            reward = np.asscalar(agent.learning_network.critic(tailing_state))
+    rewards = []
+    for r in reversed(batch_rewards):
+        reward = r + agent.discount * reward
+        rewards.append(reward)
+    return rewards

main.py

@@ -64,8 +64,26 @@ def dqn_cart_pole():
     agent = DQNAgent(**config)
     agent.run()
 
+def actor_critic_cart_pole():
+    config = dict()
+    config['task_fn'] = lambda: CartPole()
+    config['optimizer_fn'] = lambda params: torch.optim.SGD(params, 0.001)
+    config['network_fn'] = lambda: ActorCriticNet([4, 200, 2])
+    config['policy_fn'] = SamplePolicy
+    config['bootstrap_fn'] = AdvantageActorCritic
+    config['discount'] = 0.99
+    config['target_network_update_freq'] = 200
+    config['step_limit'] = 300
+    config['n_workers'] = 8
+    config['batch_size'] = 5
+    config['test_interval'] = 500
+    config['test_repeats'] = 5
+    agent = AsyncAgent(**config)
+    agent.run()
+
 if __name__ == '__main__':
-    async_cart_pole()
+    # async_cart_pole()
     # async_lunar_lander()
     # dqn_cart_pole()
     # dqn_mountain_car()
+    actor_critic_cart_pole()

network.py

@@ -64,3 +64,43 @@ class FullyConnectedNet(nn.Module):
 
     def output_transfer(self, y):
         return y
+
+class ActorCriticNet(nn.Module):
+    def __init__(self, dims, gpu=True):
+        super(ActorCriticNet, self).__init__()
+        self.fc1 = nn.Linear(dims[0], dims[1])
+        self.fc_actor = nn.Linear(dims[1], dims[2])
+        self.fc_critic = nn.Linear(dims[1], 1)
+        self.gpu = gpu and torch.cuda.is_available()
+        if self.gpu:
+            print 'Transferring network to GPU...'
+            self.cuda()
+            print 'Network transferred.'
+
+    def forward(self, x):
+        x = torch.from_numpy(np.asarray(x, dtype='float32'))
+        if self.gpu:
+            x = x.cuda()
+        x = Variable(x)
+        phi = self.fc1(x)
+        return phi
+
+    def predict(self, x):
+        phi = self.forward(x)
+        return F.softmax(self.fc_actor(phi)).cpu().data.numpy()
+
+    def gradient(self, x, actions, rewards):
+        phi = self.forward(x)
+        logit = self.fc_actor(phi)
+        log_prob = F.log_softmax(logit)
+        state_value = self.fc_critic(phi)
+        log_prob = log_prob.gather(1, Variable(torch.from_numpy(np.asarray([actions]).reshape([-1, 1]))))
+        advantage = np.asarray([rewards]).reshape([-1, 1]) - state_value.cpu().data.numpy()
+        policy_loss = -torch.sum(log_prob * Variable(torch.from_numpy(np.asarray(advantage, dtype='float32'))))
+        value_loss = 0.5 * torch.sum(torch.pow(state_value - Variable(torch.from_numpy(np.asarray(rewards, dtype='float32'))), 2))
+        (policy_loss + value_loss).backward()
+        nn.utils.clip_grad_norm(self.parameters(), 40)
+
+    def critic(self, x):
+        phi = self.forward(x)
+        return self.fc_critic(phi).cpu().data.numpy()

policy.py

@@ -13,12 +13,18 @@ class GreedyPolicy:
         self.min_epsilon = min_epsilon
         self.end_episode = end_episode
 
-    def sample(self, state):
+    def sample(self, action_value):
         if np.random.rand() < self.epsilon:
-            return np.random.randint(0, len(state))
-        return np.argmax(state)
+            return np.random.randint(0, len(action_value))
+        return np.argmax(action_value)
 
     def update_epsilon(self):
         self.epsilon = self.init_epsilon - float(self.current_episode) / self.end_episode * (self.init_epsilon - self.min_epsilon)
         self.epsilon = max(self.epsilon, self.min_epsilon)
         self.current_episode += 1
+
+class SamplePolicy:
+    def sample(self, action_value):
+        return np.random.choice(np.arange(len(action_value)), p=action_value)
+    def update_epsilon(self):
+        pass