wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-08-24 11:29:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Double DQN and Dueling DQN

Browse Source
This commit is contained in:
Shangtong Zhang
2017-06-03 15:03:17 -06:00
parent 52918587be
commit 353541cd32
5 changed files with 88 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -11,6 +11,7 @@ data
draw_*
log
figure
to_plot
# C extensions
*.so
README.md
+8 -4
View File
@@ -3,10 +3,12 @@ Highly modularized implementation of popular deep RL algorithms by PyTorch. My p
reuse as much components as I can through different algorithms, use as less tricks as I can and switch
easily between classical control tasks like CartPole and Atari games with raw pixel inputs.
* Deep Q-Learning (DQN)
* Asynchronous One-Step Q-Learning
* Asynchronous One-Step Sarsa
* Asynchronous N-Step Q-Learning
* Asynchronous Advantage Actor Critic (A3C)
* Double DQN
* Dueling DQN
* Async One-Step Q-Learning
* Async One-Step Sarsa
* Async N-Step Q-Learning
* Async Advantage Actor Critic (A3C)
# Curves
> Curves for CartPole is trivial so I didn't place it here.
@@ -46,5 +48,7 @@ Detailed usage and all training details can be found in ```main.py```
# References
* [Human Level Control through Deep Reinforcement Learning](https://www.nature.com/nature/journal/v518/n7540/full/nature14236.html)
* [Asynchronous Methods for Deep Reinforcement Learning](https://arxiv.org/abs/1602.01783)
* [Deep Reinforcement Learning with Double Q-learning](https://arxiv.org/abs/1509.06461)
* [Dueling Network Architectures for Deep Reinforcement Learning](https://arxiv.org/abs/1511.06581)
* [transedward/pytorch-dqn](https://github.com/transedward/pytorch-dqn)
* [ikostrikov/pytorch-a3c](https://github.com/ikostrikov/pytorch-a3c)
dqn_agent.py
+12 -8
View File
@@ -9,7 +9,6 @@ from replay import *
from policy import *
import numpy as np
import time
import psutil
import os
import pickle
@@ -25,6 +24,7 @@ class DQNAgent:
target_network_update_freq,
explore_steps,
history_length,
double_q,
test_interval,
test_repetitions,
logger):
@@ -41,10 +41,11 @@ class DQNAgent:
self.explore_steps = explore_steps
self.history_length = history_length
self.logger = logger
self.process = psutil.Process(os.getpid())
self.test_interval = test_interval
self.test_repetitions = test_repetitions
self.history_buffer = None
self.double_q = double_q
self.tag = ''
def episode(self, deterministic=False):
episode_start_time = time.time()
@@ -81,7 +82,11 @@ class DQNAgent:
states = self.task.normalize_state(states)
next_states = self.task.normalize_state(next_states)
q_next = self.target_network.predict(next_states, False).detach()
q_next, _ = q_next.max(1)
if self.double_q:
_, best_actions = self.learning_network.predict(next_states, False).detach().max(1)
q_next = q_next.gather(1, best_actions)
else:
q_next, _ = q_next.max(1)
terminals = self.learning_network.to_torch_variable(terminals).unsqueeze(1)
rewards = self.learning_network.to_torch_variable(rewards).unsqueeze(1)
q_next = q_next * (1 - terminals)
@@ -98,9 +103,8 @@ class DQNAgent:
if not deterministic and self.total_steps > self.explore_steps:
self.policy.update_epsilon()
episode_time = time.time() - episode_start_time
info = self.process.memory_info()
self.logger.debug('episode steps %d, episode time %f, time per step %f, rss %d, vms %d' %
(steps, episode_time, episode_time / float(steps), info.rss, info.vms))
self.logger.debug('episode steps %d, episode time %f, time per step %f' %
(steps, episode_time, episode_time / float(steps)))
return total_reward
def save(self, file_name):
@@ -122,7 +126,7 @@ class DQNAgent:
if ep % self.test_interval == 0:
self.logger.info('Testing...')
self.save('data/dqn-model-%s.bin' % (self.task.name))
self.save('data/%sdqn-model-%s.bin' % (self.tag, self.task.name))
test_rewards = []
for _ in range(self.test_repetitions):
test_rewards.append(self.episode(True))
@@ -130,7 +134,7 @@ class DQNAgent:
avg_test_rewards.append(avg_reward)
self.logger.info('Avg reward %f(%f)' % (
avg_reward, np.std(test_rewards) / np.sqrt(self.test_repetitions)))
with open('data/dqn-statistics-%s.bin' % (self.task.name), 'wb') as f:
with open('data/%sdqn-statistics-%s.bin' % (self.tag, self.task.name), 'wb') as f:
pickle.dump({'rewards': rewards,
'test_rewards': avg_test_rewards}, f)
if avg_reward > self.task.success_threshold:
main.py
+16 -8
View File
@@ -6,7 +6,8 @@ def dqn_cart_pole():
config = dict()
config['task_fn'] = lambda: CartPole()
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001)
config['network_fn'] = lambda optimizer_fn: FullyConnectedNet([8, 50, 200, 2], optimizer_fn)
# config['network_fn'] = lambda optimizer_fn: FullyConnectedNet([8, 50, 200, 2], optimizer_fn)
config['network_fn'] = lambda optimizer_fn: DuelingFullyConnectedNet([8, 50, 200, 2], optimizer_fn)
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1)
config['replay_fn'] = lambda: Replay(memory_size=10000, batch_size=10)
config['discount'] = 0.99
@@ -17,6 +18,8 @@ def dqn_cart_pole():
config['history_length'] = 2
config['test_interval'] = 100
config['test_repetitions'] = 50
# config['double_q'] = True
config['double_q'] = False
agent = DQNAgent(**config)
agent.run()
@@ -66,7 +69,8 @@ def dqn_pixel_atari(name):
n_actions = 6
config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, normalized_state=False)
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01)
config['network_fn'] = lambda optimizer_fn: ConvNet(history_length, n_actions, optimizer_fn)
# config['network_fn'] = lambda optimizer_fn: ConvNet(history_length, n_actions, optimizer_fn)
config['network_fn'] = lambda optimizer_fn: DuelingConvNet(history_length, n_actions, optimizer_fn)
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
config['replay_fn'] = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8)
config['discount'] = 0.99
@@ -75,9 +79,12 @@ def dqn_pixel_atari(name):
config['explore_steps'] = 50000
config['logger'] = gym.logger
config['history_length'] = history_length
config['test_interval'] = 1000
config['test_repetitions'] = 50
config['test_interval'] = 10
config['test_repetitions'] = 1
# config['double_q'] = True
config['double_q'] = False
agent = DQNAgent(**config)
agent.tag = 'dueling_'
agent.run()
def async_pixel_atari(name):
@@ -88,9 +95,9 @@ def async_pixel_atari(name):
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001)
config['network_fn'] = lambda: ConvNet(history_length, n_actions, gpu=False)
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
config['bootstrap_fn'] = OneStepQLearning
# config['bootstrap_fn'] = OneStepQLearning
# config['bootstrap_fn'] = NStepQLearning
# config['bootstrap_fn'] = OneStepSarsa
config['bootstrap_fn'] = OneStepSarsa
config['discount'] = 0.99
config['target_network_update_freq'] = 10000
config['step_limit'] = 10000
@@ -129,10 +136,11 @@ if __name__ == '__main__':
gym.logger.setLevel(logging.INFO)
# async_cart_pole()
# dqn_cart_pole()
dqn_cart_pole()
# dqn_pixel_atari('BreakoutNoFrameskip-v3')
# async_pixel_atari('BreakoutNoFrameskip-v3')
# a3c_pixel_atari('BreakoutNoFrameskip-v3')
# a3c_cart_pole()
async_pixel_atari('PongNoFrameskip-v3')
# dqn_pixel_atari('PongNoFrameskip-v3')
# async_pixel_atari('PongNoFrameskip-v3')
# a3c_pixel_atari('PongNoFrameskip-v3')
network.py
+51
View File
@@ -66,6 +66,17 @@ class ActorCriticNet(BasicNet):
phi = self.forward(x)
return self.fc_critic(phi).cpu().data.numpy()
# Base class for dueling architecture
class DuelingNet(BasicNet):
def predict(self, x, to_numpy=True):
phi = self.forward(x)
value = self.fc_value(phi)
advantange = self.fc_advantage(phi)
q = value.expand_as(advantange) + (advantange - advantange.mean(1).expand_as(advantange))
if to_numpy:
return q.cpu().data.numpy()
return q
# Network for CartPole with value based methods
class FullyConnectedNet(nn.Module, VanillaNet):
def __init__(self, dims, optimizer_fn=None, gpu=True):
@@ -84,6 +95,24 @@ class FullyConnectedNet(nn.Module, VanillaNet):
y = self.fc3(y)
return y
# Network for CartPole with dueling architecture
class DuelingFullyConnectedNet(nn.Module, DuelingNet):
def __init__(self, dims, optimizer_fn=None, gpu=True):
super(DuelingFullyConnectedNet, self).__init__()
self.fc1 = nn.Linear(dims[0], dims[1])
self.fc2 = nn.Linear(dims[1], dims[2])
self.fc_value = nn.Linear(dims[2], 1)
self.fc_advantage = nn.Linear(dims[2], dims[3])
self.criterion = nn.MSELoss()
BasicNet.__init__(self, optimizer_fn, gpu)
def forward(self, x):
x = self.to_torch_variable(x)
x = x.view(x.size(0), -1)
y = F.relu(self.fc1(x))
phi = F.relu(self.fc2(y))
return phi
# Network for pixel Atari game with value based methods
class ConvNet(nn.Module, VanillaNet):
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
@@ -105,6 +134,28 @@ class ConvNet(nn.Module, VanillaNet):
y = F.relu(self.fc4(y))
return self.fc5(y)
# Network for pixel Atari game with dueling architecture
class DuelingConvNet(nn.Module, DuelingNet):
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
super(DuelingConvNet, self).__init__()
self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)
self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
self.fc4 = nn.Linear(7 * 7 * 64, 512)
self.fc_advantage = nn.Linear(512, n_actions)
self.fc_value = nn.Linear(512, 1)
self.criterion = nn.MSELoss()
BasicNet.__init__(self, optimizer_fn, gpu)
def forward(self, x):
x = self.to_torch_variable(x)
y = F.relu(self.conv1(x))
y = F.relu(self.conv2(y))
y = F.relu(self.conv3(y))
y = y.view(y.size(0), -1)
phi = F.relu(self.fc4(y))
return phi
# Network for CartPole with actor critic
class FCActorCriticNet(nn.Module, ActorCriticNet):
def __init__(self,