diff --git a/.gitignore b/.gitignore index 0347215..66f6f39 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ data draw_* log figure +to_plot # C extensions *.so diff --git a/README.md b/README.md index 3b9899f..eefb1d2 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,12 @@ Highly modularized implementation of popular deep RL algorithms by PyTorch. My p reuse as much components as I can through different algorithms, use as less tricks as I can and switch easily between classical control tasks like CartPole and Atari games with raw pixel inputs. * Deep Q-Learning (DQN) -* Asynchronous One-Step Q-Learning -* Asynchronous One-Step Sarsa -* Asynchronous N-Step Q-Learning -* Asynchronous Advantage Actor Critic (A3C) +* Double DQN +* Dueling DQN +* Async One-Step Q-Learning +* Async One-Step Sarsa +* Async N-Step Q-Learning +* Async Advantage Actor Critic (A3C) # Curves > Curves for CartPole is trivial so I didn't place it here. @@ -46,5 +48,7 @@ Detailed usage and all training details can be found in ```main.py``` # References * [Human Level Control through Deep Reinforcement Learning](https://www.nature.com/nature/journal/v518/n7540/full/nature14236.html) * [Asynchronous Methods for Deep Reinforcement Learning](https://arxiv.org/abs/1602.01783) +* [Deep Reinforcement Learning with Double Q-learning](https://arxiv.org/abs/1509.06461) +* [Dueling Network Architectures for Deep Reinforcement Learning](https://arxiv.org/abs/1511.06581) * [transedward/pytorch-dqn](https://github.com/transedward/pytorch-dqn) * [ikostrikov/pytorch-a3c](https://github.com/ikostrikov/pytorch-a3c) diff --git a/dqn_agent.py b/dqn_agent.py index f614323..13acc59 100644 --- a/dqn_agent.py +++ b/dqn_agent.py @@ -9,7 +9,6 @@ from replay import * from policy import * import numpy as np import time -import psutil import os import pickle @@ -25,6 +24,7 @@ class DQNAgent: target_network_update_freq, explore_steps, history_length, + double_q, test_interval, test_repetitions, logger): @@ -41,10 +41,11 @@ class DQNAgent: self.explore_steps = explore_steps self.history_length = history_length self.logger = logger - self.process = psutil.Process(os.getpid()) self.test_interval = test_interval self.test_repetitions = test_repetitions self.history_buffer = None + self.double_q = double_q + self.tag = '' def episode(self, deterministic=False): episode_start_time = time.time() @@ -81,7 +82,11 @@ class DQNAgent: states = self.task.normalize_state(states) next_states = self.task.normalize_state(next_states) q_next = self.target_network.predict(next_states, False).detach() - q_next, _ = q_next.max(1) + if self.double_q: + _, best_actions = self.learning_network.predict(next_states, False).detach().max(1) + q_next = q_next.gather(1, best_actions) + else: + q_next, _ = q_next.max(1) terminals = self.learning_network.to_torch_variable(terminals).unsqueeze(1) rewards = self.learning_network.to_torch_variable(rewards).unsqueeze(1) q_next = q_next * (1 - terminals) @@ -98,9 +103,8 @@ class DQNAgent: if not deterministic and self.total_steps > self.explore_steps: self.policy.update_epsilon() episode_time = time.time() - episode_start_time - info = self.process.memory_info() - self.logger.debug('episode steps %d, episode time %f, time per step %f, rss %d, vms %d' % - (steps, episode_time, episode_time / float(steps), info.rss, info.vms)) + self.logger.debug('episode steps %d, episode time %f, time per step %f' % + (steps, episode_time, episode_time / float(steps))) return total_reward def save(self, file_name): @@ -122,7 +126,7 @@ class DQNAgent: if ep % self.test_interval == 0: self.logger.info('Testing...') - self.save('data/dqn-model-%s.bin' % (self.task.name)) + self.save('data/%sdqn-model-%s.bin' % (self.tag, self.task.name)) test_rewards = [] for _ in range(self.test_repetitions): test_rewards.append(self.episode(True)) @@ -130,7 +134,7 @@ class DQNAgent: avg_test_rewards.append(avg_reward) self.logger.info('Avg reward %f(%f)' % ( avg_reward, np.std(test_rewards) / np.sqrt(self.test_repetitions))) - with open('data/dqn-statistics-%s.bin' % (self.task.name), 'wb') as f: + with open('data/%sdqn-statistics-%s.bin' % (self.tag, self.task.name), 'wb') as f: pickle.dump({'rewards': rewards, 'test_rewards': avg_test_rewards}, f) if avg_reward > self.task.success_threshold: diff --git a/main.py b/main.py index a6326f1..83a5ca2 100644 --- a/main.py +++ b/main.py @@ -6,7 +6,8 @@ def dqn_cart_pole(): config = dict() config['task_fn'] = lambda: CartPole() config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001) - config['network_fn'] = lambda optimizer_fn: FullyConnectedNet([8, 50, 200, 2], optimizer_fn) + # config['network_fn'] = lambda optimizer_fn: FullyConnectedNet([8, 50, 200, 2], optimizer_fn) + config['network_fn'] = lambda optimizer_fn: DuelingFullyConnectedNet([8, 50, 200, 2], optimizer_fn) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1) config['replay_fn'] = lambda: Replay(memory_size=10000, batch_size=10) config['discount'] = 0.99 @@ -17,6 +18,8 @@ def dqn_cart_pole(): config['history_length'] = 2 config['test_interval'] = 100 config['test_repetitions'] = 50 + # config['double_q'] = True + config['double_q'] = False agent = DQNAgent(**config) agent.run() @@ -66,7 +69,8 @@ def dqn_pixel_atari(name): n_actions = 6 config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, normalized_state=False) config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) - config['network_fn'] = lambda optimizer_fn: ConvNet(history_length, n_actions, optimizer_fn) + # config['network_fn'] = lambda optimizer_fn: ConvNet(history_length, n_actions, optimizer_fn) + config['network_fn'] = lambda optimizer_fn: DuelingConvNet(history_length, n_actions, optimizer_fn) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) config['replay_fn'] = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8) config['discount'] = 0.99 @@ -75,9 +79,12 @@ def dqn_pixel_atari(name): config['explore_steps'] = 50000 config['logger'] = gym.logger config['history_length'] = history_length - config['test_interval'] = 1000 - config['test_repetitions'] = 50 + config['test_interval'] = 10 + config['test_repetitions'] = 1 + # config['double_q'] = True + config['double_q'] = False agent = DQNAgent(**config) + agent.tag = 'dueling_' agent.run() def async_pixel_atari(name): @@ -88,9 +95,9 @@ def async_pixel_atari(name): config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001) config['network_fn'] = lambda: ConvNet(history_length, n_actions, gpu=False) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) - config['bootstrap_fn'] = OneStepQLearning + # config['bootstrap_fn'] = OneStepQLearning # config['bootstrap_fn'] = NStepQLearning - # config['bootstrap_fn'] = OneStepSarsa + config['bootstrap_fn'] = OneStepSarsa config['discount'] = 0.99 config['target_network_update_freq'] = 10000 config['step_limit'] = 10000 @@ -129,10 +136,11 @@ if __name__ == '__main__': gym.logger.setLevel(logging.INFO) # async_cart_pole() - # dqn_cart_pole() + dqn_cart_pole() # dqn_pixel_atari('BreakoutNoFrameskip-v3') # async_pixel_atari('BreakoutNoFrameskip-v3') # a3c_pixel_atari('BreakoutNoFrameskip-v3') # a3c_cart_pole() - async_pixel_atari('PongNoFrameskip-v3') + # dqn_pixel_atari('PongNoFrameskip-v3') + # async_pixel_atari('PongNoFrameskip-v3') # a3c_pixel_atari('PongNoFrameskip-v3') diff --git a/network.py b/network.py index 45fdfff..e13467d 100644 --- a/network.py +++ b/network.py @@ -66,6 +66,17 @@ class ActorCriticNet(BasicNet): phi = self.forward(x) return self.fc_critic(phi).cpu().data.numpy() +# Base class for dueling architecture +class DuelingNet(BasicNet): + def predict(self, x, to_numpy=True): + phi = self.forward(x) + value = self.fc_value(phi) + advantange = self.fc_advantage(phi) + q = value.expand_as(advantange) + (advantange - advantange.mean(1).expand_as(advantange)) + if to_numpy: + return q.cpu().data.numpy() + return q + # Network for CartPole with value based methods class FullyConnectedNet(nn.Module, VanillaNet): def __init__(self, dims, optimizer_fn=None, gpu=True): @@ -84,6 +95,24 @@ class FullyConnectedNet(nn.Module, VanillaNet): y = self.fc3(y) return y +# Network for CartPole with dueling architecture +class DuelingFullyConnectedNet(nn.Module, DuelingNet): + def __init__(self, dims, optimizer_fn=None, gpu=True): + super(DuelingFullyConnectedNet, self).__init__() + self.fc1 = nn.Linear(dims[0], dims[1]) + self.fc2 = nn.Linear(dims[1], dims[2]) + self.fc_value = nn.Linear(dims[2], 1) + self.fc_advantage = nn.Linear(dims[2], dims[3]) + self.criterion = nn.MSELoss() + BasicNet.__init__(self, optimizer_fn, gpu) + + def forward(self, x): + x = self.to_torch_variable(x) + x = x.view(x.size(0), -1) + y = F.relu(self.fc1(x)) + phi = F.relu(self.fc2(y)) + return phi + # Network for pixel Atari game with value based methods class ConvNet(nn.Module, VanillaNet): def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True): @@ -105,6 +134,28 @@ class ConvNet(nn.Module, VanillaNet): y = F.relu(self.fc4(y)) return self.fc5(y) +# Network for pixel Atari game with dueling architecture +class DuelingConvNet(nn.Module, DuelingNet): + def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True): + super(DuelingConvNet, self).__init__() + self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4) + self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2) + self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1) + self.fc4 = nn.Linear(7 * 7 * 64, 512) + self.fc_advantage = nn.Linear(512, n_actions) + self.fc_value = nn.Linear(512, 1) + self.criterion = nn.MSELoss() + BasicNet.__init__(self, optimizer_fn, gpu) + + def forward(self, x): + x = self.to_torch_variable(x) + y = F.relu(self.conv1(x)) + y = F.relu(self.conv2(y)) + y = F.relu(self.conv3(y)) + y = y.view(y.size(0), -1) + phi = F.relu(self.fc4(y)) + return phi + # Network for CartPole with actor critic class FCActorCriticNet(nn.Module, ActorCriticNet): def __init__(self,