wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-08-28 12:42:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Finalize async methods

Browse Source
This commit is contained in:
Shangtong Zhang
2017-06-10 21:08:40 -06:00
parent 9c73abfbff
commit 8ea108e4bb
6 changed files with 76 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
async_agent.py
+8 -7
View File
@@ -59,6 +59,7 @@ class AsyncAgent:
self.test_repetitions = test_repetitions
self.logger = logger
self.history_length = history_length
self.tag = ''
def deterministic_episode(self, task, network):
state = task.reset()
@@ -73,12 +74,14 @@ class AsyncAgent:
total_rewards += reward
if terminal:
break
bootstrap.reset()
return total_rewards
def worker(self, id):
optimizer = self.optimizer_fn(self.learning_network.parameters())
worker_network = self.network_fn()
worker_network.load_state_dict(self.learning_network.state_dict())
bootstrap = self.bootstrap(self)
task = self.task_fn()
policy = self.policy_fn()
@@ -125,8 +128,7 @@ class AsyncAgent:
state = next_state
if self.target_network and self.total_steps.value % self.target_network_update_freq == 0:
with self.network_lock:
self.target_network.load_state_dict(self.learning_network.state_dict())
self.target_network.load_state_dict(self.learning_network.state_dict())
def save(self, file_name):
with open(file_name, 'wb') as f:
@@ -142,9 +144,8 @@ class AsyncAgent:
while True:
steps = self.total_steps.value + 1
if steps % self.test_interval == 0:
with self.network_lock:
test_network.load_state_dict(self.learning_network.state_dict())
self.save('data/%s-model-%s.bin' % (self.bootstrap.__name__, self.task.name))
test_network.load_state_dict(self.learning_network.state_dict())
self.save('data/%s%s-model-%s.bin' % (self.tag, self.bootstrap.__name__, self.task.name))
rewards = np.zeros(self.test_repetitions)
for i in range(self.test_repetitions):
rewards[i] = self.deterministic_episode(self.task, test_network)
@@ -152,8 +153,8 @@ class AsyncAgent:
(steps, np.mean(rewards), np.std(rewards) / np.sqrt(self.test_repetitions)))
test_rewards.append(np.mean(rewards))
test_points.append(steps)
with open('data/%s-statistics-%s.bin' % (
self.bootstrap.__name__, self.task.name
with open('data/%s%s-statistics-%s.bin' % (
self.tag, self.bootstrap.__name__, self.task.name
), 'wb') as f:
pickle.dump([test_points, test_rewards], f)
if np.mean(rewards) > self.task.success_threshold:
atari_wrapper.py
+1 -1
View File
@@ -134,7 +134,7 @@ class ProcessFrame(gym.Wrapper):
elif frame_size == 42:
self.process_fn = _process_frame42
else:
assert(False, "Unknown frame size")
assert False, "Unknown frame size"
def _step(self, action):
obs, reward, done, info = self.env.step(action)
bootstrap.py
+16 -4
View File
@@ -10,6 +10,9 @@ from torch.autograd import Variable
class OneStepSarsa:
def __init__(self, agent):
self.agent = agent
self.reset()
def reset(self):
self.pending = []
def process_state(self, network, state):
@@ -36,13 +39,16 @@ class OneStepSarsa:
q_next = self.agent.discount * q_next + reward
q = q.gather(1, Variable(torch.LongTensor([[action]])))
loss += 0.5 * (q - Variable(q_next)).pow(2)
self.pending = []
self.reset()
return loss
class OneStepQLearning:
def __init__(self, agent):
self.agent = agent
self.reset()
def reset(self):
self.pending = []
def process_state(self, network, state):
@@ -63,12 +69,15 @@ class OneStepQLearning:
q_next = self.agent.discount * q_next + reward
q = q.gather(1, Variable(torch.LongTensor([[action]])))
loss += 0.5 * (q - Variable(q_next)).pow(2)
self.pending = []
self.reset()
return loss
class NStepQLearning:
def __init__(self, agent):
self.agent = agent
self.reset()
def reset(self):
self.pending = []
def process_state(self, network, state):
@@ -92,12 +101,15 @@ class NStepQLearning:
q, action, reward = self.pending[i]
R = reward + self.agent.discount * R
loss += 0.5 * (Variable(R) - q.gather(1, Variable(torch.LongTensor([[action]])))).pow(2)
self.pending = []
self.reset()
return loss
class AdvantageActorCritic:
def __init__(self, agent):
self.agent = agent
self.reset()
def reset(self):
self.pending = []
def process_state(self, network, state):
@@ -122,6 +134,6 @@ class AdvantageActorCritic:
loss += 0.5 * advantage.pow(2)
loss += -log_prob.gather(1, Variable(torch.LongTensor([[action]]))) * Variable(advantage.data)
loss += 0.01 * torch.sum(torch.mul(prob, log_prob))
self.pending = []
self.reset()
return loss
images/A3C-PongNoFrameskip-v3.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 30 KiB

main.py
+18 -18
View File
@@ -6,8 +6,8 @@ def dqn_cart_pole():
config = dict()
config['task_fn'] = lambda: CartPole()
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001)
config['network_fn'] = lambda optimizer_fn: FullyConnectedNet([8, 50, 200, 2], optimizer_fn)
# config['network_fn'] = lambda optimizer_fn: DuelingFullyConnectedNet([8, 50, 200, 2], optimizer_fn)
config['network_fn'] = lambda optimizer_fn: FCNet([8, 50, 200, 2], optimizer_fn)
# config['network_fn'] = lambda optimizer_fn: DuelingFCNet([8, 50, 200, 2], optimizer_fn)
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1)
config['replay_fn'] = lambda: Replay(memory_size=10000, batch_size=10)
config['discount'] = 0.99
@@ -27,7 +27,7 @@ def async_cart_pole():
config = dict()
config['task_fn'] = lambda: CartPole()
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001)
config['network_fn'] = lambda: FullyConnectedNet([4, 50, 200, 2])
config['network_fn'] = lambda: FCNet([4, 50, 200, 2])
config['policy_fn'] = lambda: GreedyPolicy(epsilon=0.5, final_step=5000, min_epsilon=0.1)
config['bootstrap'] = OneStepQLearning
# config['bootstrap'] = NStepQLearning
@@ -49,7 +49,7 @@ def a3c_cart_pole():
config = dict()
config['task_fn'] = lambda: CartPole()
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001)
config['network_fn'] = lambda: FCActorCriticNet([4, 200, 2])
config['network_fn'] = lambda: ActorCriticFCNet([4, 200, 2])
config['policy_fn'] = SamplePolicy
config['bootstrap'] = AdvantageActorCritic
config['discount'] = 0.99
@@ -70,8 +70,8 @@ def dqn_pixel_atari(name):
n_actions = 6
config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, normalized_state=False)
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01)
# config['network_fn'] = lambda optimizer_fn: ConvNet(history_length, n_actions, optimizer_fn)
config['network_fn'] = lambda optimizer_fn: DuelingConvNet(history_length, n_actions, optimizer_fn)
config['network_fn'] = lambda optimizer_fn: NatureConvNet(history_length, n_actions, optimizer_fn)
# config['network_fn'] = lambda optimizer_fn: DuelingNatureConvNet(history_length, n_actions, optimizer_fn)
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
config['replay_fn'] = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8)
config['discount'] = 0.99
@@ -95,20 +95,19 @@ def async_pixel_atari(name):
config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, frame_size=42)
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001)
config['network_fn'] = lambda: OpenAIConvNet(history_length,
n_actions,
LSTM=False)
config['policy_fn'] = lambda: StochasticGreedyPolicy(epsilons=[1.0, 1.0, 1.0],
final_step=1000000,
min_epsilons=[0.1, 0.01, 0.5],
n_actions)
config['policy_fn'] = lambda: StochasticGreedyPolicy(epsilons=[0.5, 0.5, 0.5],
final_step=2000000,
min_epsilons=[0.1, 0.01, 0.2],
probs=[0.4, 0.3, 0.3])
# config['bootstrap'] = OneStepQLearning
config['bootstrap'] = NStepQLearning
# config['bootstrap'] = OneStepSarsa
# config['bootstrap'] = NStepQLearning
config['bootstrap'] = OneStepSarsa
config['discount'] = 0.99
config['target_network_update_freq'] = 10000
config['step_limit'] = 10000
config['n_workers'] = 16
config['update_interval'] = 32
config['update_interval'] = 20
config['test_interval'] = 50000
config['test_repetitions'] = 1
config['history_length'] = history_length
@@ -122,9 +121,9 @@ def a3c_pixel_atari(name):
n_actions = 6
config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, frame_size=42)
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001)
config['network_fn'] = lambda: OpenAIConvActorCriticNet(history_length,
config['network_fn'] = lambda: OpenAIActorCriticConvNet(history_length,
n_actions,
LSTM=True)
LSTM=False)
config['policy_fn'] = SamplePolicy
config['bootstrap'] = AdvantageActorCritic
config['discount'] = 0.99
@@ -137,6 +136,7 @@ def a3c_pixel_atari(name):
config['history_length'] = history_length
config['logger'] = gym.logger
agent = AsyncAgent(**config)
agent.tag = ''
agent.run()
if __name__ == '__main__':
@@ -148,9 +148,9 @@ if __name__ == '__main__':
# a3c_cart_pole()
# dqn_pixel_atari('PongNoFrameskip-v3')
# async_pixel_atari('PongNoFrameskip-v3')
async_pixel_atari('PongNoFrameskip-v3')
# a3c_pixel_atari('PongNoFrameskip-v3')
# dqn_pixel_atari('BreakoutNoFrameskip-v3')
async_pixel_atari('BreakoutNoFrameskip-v3')
# async_pixel_atari('BreakoutNoFrameskip-v3')
# a3c_pixel_atari('BreakoutNoFrameskip-v3')
network.py
+33 -81
View File
@@ -18,9 +18,7 @@ class BasicNet:
self.gpu = gpu and torch.cuda.is_available()
self.LSTM = LSTM
if self.gpu:
print 'Transferring network to GPU...'
self.cuda()
print 'Network transferred.'
def to_torch_variable(self, x, dtype='float32'):
if not isinstance(x, torch.FloatTensor):
@@ -74,9 +72,9 @@ class DuelingNet(BasicNet):
# Starting of several network instances
# Network for CartPole with value based methods
class FullyConnectedNet(nn.Module, VanillaNet):
class FCNet(nn.Module, VanillaNet):
def __init__(self, dims, optimizer_fn=None, gpu=True):
super(FullyConnectedNet, self).__init__()
super(FCNet, self).__init__()
self.fc1 = nn.Linear(dims[0], dims[1])
self.fc2 = nn.Linear(dims[1], dims[2])
self.fc3 = nn.Linear(dims[2], dims[3])
@@ -92,9 +90,9 @@ class FullyConnectedNet(nn.Module, VanillaNet):
return y
# Network for CartPole with dueling architecture
class DuelingFullyConnectedNet(nn.Module, DuelingNet):
class DuelingFCNet(nn.Module, DuelingNet):
def __init__(self, dims, optimizer_fn=None, gpu=True):
super(DuelingFullyConnectedNet, self).__init__()
super(DuelingFCNet, self).__init__()
self.fc1 = nn.Linear(dims[0], dims[1])
self.fc2 = nn.Linear(dims[1], dims[2])
self.fc_value = nn.Linear(dims[2], 1)
@@ -109,10 +107,26 @@ class DuelingFullyConnectedNet(nn.Module, DuelingNet):
phi = F.relu(self.fc2(y))
return phi
# Network for CartPole with actor critic
class ActorCriticFCNet(nn.Module, ActorCriticNet):
def __init__(self,
dims):
super(ActorCriticFCNet, self).__init__()
self.layer1 = nn.Linear(dims[0], dims[1])
self.fc_actor = nn.Linear(dims[1], dims[2])
self.fc_critic = nn.Linear(dims[1], 1)
BasicNet.__init__(self, None, False)
def forward(self, x, update_LSTM=True):
x = self.to_torch_variable(x)
x = x.view(x.size(0), -1)
phi = self.layer1(x)
return phi
# Network for pixel Atari game with value based methods
class ConvNet(nn.Module, VanillaNet):
class NatureConvNet(nn.Module, VanillaNet):
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
super(ConvNet, self).__init__()
super(NatureConvNet, self).__init__()
self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)
self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
@@ -130,28 +144,10 @@ class ConvNet(nn.Module, VanillaNet):
y = F.relu(self.fc4(y))
return self.fc5(y)
class NipsConvNet(nn.Module, VanillaNet):
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
super(NipsConvNet, self).__init__()
self.conv1 = nn.Conv2d(in_channels, 16, kernel_size=8, stride=4)
self.conv2 = nn.Conv2d(16, 32, kernel_size=4, stride=2)
self.fc3 = nn.Linear(9 * 9 * 32, 256)
self.fc4 = nn.Linear(256, n_actions)
self.criterion = nn.MSELoss()
BasicNet.__init__(self, optimizer_fn, gpu)
def forward(self, x):
x = self.to_torch_variable(x)
y = F.relu(self.conv1(x))
y = F.relu(self.conv2(y))
y = y.view(y.size(0), -1)
y = F.relu(self.fc3(y))
return self.fc4(y)
# Network for pixel Atari game with dueling architecture
class DuelingConvNet(nn.Module, DuelingNet):
class DuelingNatureConvNet(nn.Module, DuelingNet):
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
super(DuelingConvNet, self).__init__()
super(DuelingNatureConvNet, self).__init__()
self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)
self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
@@ -170,45 +166,17 @@ class DuelingConvNet(nn.Module, DuelingNet):
phi = F.relu(self.fc4(y))
return phi
# Network for CartPole with actor critic
class FCActorCriticNet(nn.Module, ActorCriticNet):
def __init__(self,
dims,
LSTM=False):
super(FCActorCriticNet, self).__init__()
if LSTM:
self.layer1 = nn.LSTMCell(dims[0], dims[1])
else:
self.layer1 = nn.Linear(dims[0], dims[1])
self.fc_actor = nn.Linear(dims[1], dims[2])
self.fc_critic = nn.Linear(dims[1], 1)
BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=LSTM)
if LSTM:
self.h = self.to_torch_variable(np.zeros((1, dims[1])))
self.c = self.to_torch_variable(np.zeros((1, dims[1])))
def forward(self, x, update_LSTM=True):
x = self.to_torch_variable(x)
x = x.view(x.size(0), -1)
if self.LSTM:
h, c = self.layer1(x, (self.h, self.c))
if update_LSTM:
self.h = h
self.c = c
phi = h
else:
phi = self.layer1(x)
return phi
# Network for pixel Atari game with actor critic
class ConvActorCriticNet(nn.Module, ActorCriticNet):
class ActorCriticNatureConvNet(nn.Module, ActorCriticNet):
def __init__(self,
in_channels,
n_actions,
xentropy_weight=0.01,
grad_threshold=40,
gpu=True):
super(ConvActorCriticNet, self).__init__()
super(ActorCriticNatureConvNet, self).__init__()
self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)
self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
@@ -227,12 +195,12 @@ class ConvActorCriticNet(nn.Module, ActorCriticNet):
y = y.view(y.size(0), -1)
return F.elu(self.fc4(y))
class OpenAIConvActorCriticNet(nn.Module, ActorCriticNet):
class OpenAIActorCriticConvNet(nn.Module, ActorCriticNet):
def __init__(self,
in_channels,
n_actions,
LSTM=False):
super(OpenAIConvActorCriticNet, self).__init__()
super(OpenAIActorCriticConvNet, self).__init__()
self.conv1 = nn.Conv2d(in_channels, 32, 3, stride=2, padding=1)
self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
@@ -273,27 +241,18 @@ class OpenAIConvActorCriticNet(nn.Module, ActorCriticNet):
class OpenAIConvNet(nn.Module, VanillaNet):
def __init__(self,
in_channels,
n_actions,
LSTM=False):
n_actions):
super(OpenAIConvNet, self).__init__()
self.conv1 = nn.Conv2d(in_channels, 32, 3, stride=2, padding=1)
self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
self.conv4 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
self.LSTM = LSTM
hidden_units = 256
if LSTM:
self.layer5 = nn.LSTMCell(32 * 3 * 3, hidden_units)
else:
self.layer5 = nn.Linear(32 * 3 * 3, hidden_units)
self.layer5 = nn.Linear(32 * 3 * 3, hidden_units)
self.fc6 = nn.Linear(hidden_units, n_actions)
BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=LSTM)
if LSTM:
self.h = self.to_torch_variable(np.zeros((1, hidden_units)))
self.c = self.to_torch_variable(np.zeros((1, hidden_units)))
BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=False)
def forward(self, x, update_LSTM=True):
x = self.to_torch_variable(x)
@@ -302,12 +261,5 @@ class OpenAIConvNet(nn.Module, VanillaNet):
y = F.elu(self.conv3(y))
y = F.elu(self.conv4(y))
y = y.view(y.size(0), -1)
if self.LSTM:
h, c = self.layer5(y, (self.h, self.c))
if update_LSTM:
self.h = h
self.c = c
phi = h
else:
phi = F.elu(self.layer5(y))
phi = F.elu(self.layer5(y))
return self.fc6(phi)