wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update async agents

Browse Source
This commit is contained in:
Shangtong Zhang
2017-05-30 23:39:26 -06:00
parent 8da13efc01
commit 774768640d
4 changed files with 65 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
async_agent.py
+9 -7
View File
@@ -60,7 +60,7 @@ class AsyncAgent:
steps = 0
terminal = False
buffer = [state] * self.history_length
while not terminal and steps < self.step_limit:
while not terminal and (not self.step_limit or steps < self.step_limit):
state = task.normalize_state(np.vstack(buffer))
action_values = network.predict(np.reshape(state, (1, ) + state.shape))
steps += 1
@@ -95,8 +95,8 @@ class AsyncAgent:
batch_states, batch_actions, batch_rewards = [], [], []
if terminal:
if id == 0:
self.logger.info('episode %d, epsilon %f, return %f, avg return %f, total steps %d' % (
episode, policy.epsilon, episode_return, np.mean(episode_returns[-100: ]),
self.logger.info('episode %d, return %f, avg return %f, total steps %d' % (
episode, episode_return, np.mean(episode_returns[-100: ]),
self.total_steps.value))
episode_steps = 0
episode_returns.append(episode_return)
@@ -106,7 +106,7 @@ class AsyncAgent:
state = task.reset()
buffer = [state] * self.history_length
state = task.normalize_state(np.vstack(buffer))
value = worker_network.predict(np.reshape(state, (1, ) + state.shape))
value = worker_network.predict(np.stack([state]))
action = policy.sample(value.flatten())
while not terminal and len(batch_states) < self.batch_size:
episode_steps += 1
@@ -120,18 +120,20 @@ class AsyncAgent:
buffer.pop(0)
buffer.append(state)
state = task.normalize_state(np.vstack(buffer))
value = worker_network.predict(np.reshape(state, (1, ) + state.shape))
value = worker_network.predict(np.stack([state]))
action = policy.sample(value.flatten())
policy.update_epsilon()
batch_rewards = self.bootstrap_fn(batch_states, batch_actions, batch_rewards,
state, action, terminal, self)
if episode_steps > self.step_limit:
if self.step_limit and episode_steps > self.step_limit:
terminal = True
worker_network.zero_grad()
worker_network.gradient(np.asarray(batch_states), batch_actions, batch_rewards)
worker_network.gradient(np.asarray(batch_states),
worker_network.to_torch_variable(batch_actions, 'int64').unsqueeze(1),
worker_network.to_torch_variable(batch_rewards).unsqueeze(1))
self.async_update(worker_network, optimizer)
worker_network.load_state_dict(self.learning_network.state_dict())
bootstrap.py
+1 -1
View File
@@ -51,7 +51,7 @@ def AdvantageActorCritic(batch_states, batch_actions, batch_rewards,
reward = 0
else:
with agent.network_lock:
reward = np.asscalar(agent.learning_network.critic(tailing_state))
reward = np.asscalar(agent.learning_network.critic(np.stack([tailing_state])))
rewards = []
for r in reversed(batch_rewards):
reward = r + agent.discount * reward
main.py
+15 -14
View File
@@ -5,15 +5,15 @@ import logging
def async_cart_pole():
config = dict()
config['task_fn'] = lambda: CartPole()
config['optimizer_fn'] = lambda params: torch.optim.SGD(params, 0.001)
config['network_fn'] = lambda: FullyConnectedNet([8, 50, 200, 2])
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001)
config['network_fn'] = lambda gpu=True: FullyConnectedNet([8, 50, 200, 2], gpu=gpu)
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=5000, min_epsilon=0.1)
config['bootstrap_fn'] = OneStepQLearning
# config['bootstrap_fn'] = NStepQLearning
# config['bootstrap_fn'] = OneStepSarsa
config['discount'] = 0.99
config['target_network_update_freq'] = 200
config['step_limit'] = 300
config['step_limit'] = 0
config['n_workers'] = 8
config['batch_size'] = 5
config['test_interval'] = 4000
@@ -27,7 +27,7 @@ def async_lunar_lander():
config = dict()
config['task_fn'] = lambda: LunarLander()
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001)
config['network_fn'] = lambda: FullyConnectedNet([8, 50, 200, 4])
config['network_fn'] = lambda gpu=True: FullyConnectedNet([8, 50, 200, 4], gpu=gpu)
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=40000, min_epsilon=0.05)
config['bootstrap_fn'] = OneStepQLearning
config['discount'] = 0.99
@@ -61,17 +61,18 @@ def dqn_cart_pole():
def actor_critic_cart_pole():
config = dict()
config['task_fn'] = lambda: CartPole()
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001)
config['network_fn'] = lambda: ActorCriticNet([4, 200, 2])
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001)
config['network_fn'] = lambda gpu=True: FCActorCriticNet([8, 200, 2], gpu=gpu)
config['policy_fn'] = SamplePolicy
config['bootstrap_fn'] = AdvantageActorCritic
config['discount'] = 0.99
config['target_network_update_freq'] = 200
config['step_limit'] = 200
config['n_workers'] = 10
config['batch_size'] = 5
config['test_interval'] = 50000
config['test_repetitions'] = 5
config['step_limit'] = 0
config['n_workers'] = 16
config['batch_size'] = 6
config['test_interval'] = 4000
config['history_length'] = 2
config['test_repetitions'] = 50
config['logger'] = gym.logger
agent = AsyncAgent(**config)
agent.run()
@@ -125,8 +126,8 @@ if __name__ == '__main__':
benchmark = gym.benchmark_spec('Atari40M')
# async_cart_pole()
# async_lunar_lander()
# actor_critic_cart_pole()
# dqn_cart_pole()
dqn_pixel_atari('BreakoutNoFrameskip-v3')
# async_lunar_lander()
dqn_cart_pole()
# dqn_pixel_atari('BreakoutNoFrameskip-v3')
# async_pixel_atari('BreakoutNoFrameskip-v3')
network.py
+40 -40
View File
@@ -26,6 +26,7 @@ class BasicNet:
x = x.cuda()
return Variable(x)
class VanillaNet(BasicNet):
def predict(self, x, to_numpy=True):
y = self.forward(x)
if to_numpy:
@@ -38,7 +39,31 @@ class BasicNet:
loss = self.criterion(y, targets)
loss.backward()
class FullyConnectedNet(nn.Module, BasicNet):
class ActorCriticNet(BasicNet):
def predict(self, x):
phi = self.forward(x)
return F.softmax(self.fc_actor(phi)).cpu().data.numpy()
def gradient(self, x, actions, rewards):
phi = self.forward(x)
logit = self.fc_actor(phi)
prob = F.softmax(logit)
log_prob_ = F.log_softmax(logit)
state_value = self.fc_critic(phi)
log_prob = log_prob_.gather(1, actions)
advantage = (rewards - state_value).detach()
policy_loss = -torch.sum(log_prob * advantage)
value_loss = 0.5 * torch.sum(torch.pow(state_value - rewards, 2))
entropy = -torch.sum(torch.mul(prob, log_prob_))
(policy_loss + value_loss - self.xentropy_weight * entropy).backward()
nn.utils.clip_grad_norm(self.parameters(), self.grad_threshold)
def critic(self, x):
phi = self.forward(x)
return self.fc_critic(phi).cpu().data.numpy()
class FullyConnectedNet(nn.Module, VanillaNet):
def __init__(self, dims, optimizer_fn=None, gpu=True):
super(FullyConnectedNet, self).__init__()
self.fc1 = nn.Linear(dims[0], dims[1])
@@ -55,52 +80,27 @@ class FullyConnectedNet(nn.Module, BasicNet):
y = self.fc3(y)
return y
class ActorCriticNet(nn.Module):
def __init__(self, dims, gpu=True):
super(ActorCriticNet, self).__init__()
class FCActorCriticNet(nn.Module, ActorCriticNet):
def __init__(self,
dims,
xentropy_weight=0.01,
grad_threshold=40,
gpu=True):
super(FCActorCriticNet, self).__init__()
self.fc1 = nn.Linear(dims[0], dims[1])
self.fc_actor = nn.Linear(dims[1], dims[2])
self.fc_critic = nn.Linear(dims[1], 1)
self.gpu = gpu and torch.cuda.is_available()
if self.gpu:
print 'Transferring network to GPU...'
self.cuda()
print 'Network transferred.'
def to_torch_variable(self, x, dtype='float32'):
x = torch.from_numpy(np.asarray(x, dtype=dtype))
if self.gpu:
x = x.cuda()
return Variable(x)
self.xentropy_weight = xentropy_weight
self.grad_threshold = grad_threshold
BasicNet.__init__(self, optimizer_fn=None, gpu=gpu)
def forward(self, x):
phi = self.fc1(self.to_torch_variable(x))
x = self.to_torch_variable(x)
x = x.view(x.size(0), -1)
phi = self.fc1(x)
return phi
def predict(self, x):
phi = self.forward(x)
return F.softmax(self.fc_actor(phi)).cpu().data.numpy()
def gradient(self, x, actions, rewards):
phi = self.forward(x)
logit = self.fc_actor(phi)
prob = F.softmax(logit)
log_prob_ = F.log_softmax(logit)
state_value = self.fc_critic(phi)
log_prob = log_prob_.gather(1, self.to_torch_variable(np.asarray([actions]).reshape([-1, 1]), 'int64'))
advantage = np.asarray([rewards]).reshape([-1, 1]) - state_value.cpu().data.numpy()
policy_loss = -torch.sum(log_prob * self.to_torch_variable(advantage))
value_loss = 0.5 * torch.sum(
torch.pow(state_value - Variable(torch.from_numpy(np.asarray(rewards, dtype='float32'))), 2))
entropy = -torch.sum(torch.mul(prob, log_prob_))
(policy_loss + value_loss - 0.01 * entropy).backward()
nn.utils.clip_grad_norm(self.parameters(), 40)
def critic(self, x):
phi = self.forward(x)
return self.fc_critic(phi).cpu().data.numpy()
class ConvNet(nn.Module, BasicNet):
class ConvNet(nn.Module, VanillaNet):
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
super(ConvNet, self).__init__()
self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)