mirror of
https://github.com/wassname/DeepRL.git
synced 2026-09-10 11:40:58 +08:00
Refactor async agents
This commit is contained in:
Shangtong Zhang
+28
-22
@@ -12,6 +12,7 @@ from task import *
|
||||
from network import *
|
||||
from bootstrap import *
|
||||
import pickle
|
||||
import os
|
||||
|
||||
class AsyncAgent:
|
||||
def __init__(self,
|
||||
@@ -30,12 +31,14 @@ class AsyncAgent:
|
||||
history_length,
|
||||
logger):
|
||||
self.network_fn = network_fn
|
||||
self.learning_network = network_fn(False)
|
||||
self.learning_network = network_fn()
|
||||
self.learning_network.share_memory()
|
||||
if bootstrap_fn != AdvantageActorCritic:
|
||||
self.target_network = network_fn(False)
|
||||
self.target_network = network_fn()
|
||||
self.target_network.share_memory()
|
||||
self.target_network.load_state_dict(self.learning_network.state_dict())
|
||||
else:
|
||||
self.target_network = None
|
||||
self.bootstrap_fn = bootstrap_fn
|
||||
|
||||
self.optimizer_fn = optimizer_fn
|
||||
@@ -63,7 +66,7 @@ class AsyncAgent:
|
||||
terminal = False
|
||||
buffer = [state] * self.history_length
|
||||
while not terminal and (not self.step_limit or steps < self.step_limit):
|
||||
state = task.normalize_state(np.vstack(buffer))
|
||||
state = np.vstack(buffer)
|
||||
action_values = network.predict(np.stack([state]))
|
||||
steps += 1
|
||||
action = np.argmax(action_values.flatten())
|
||||
@@ -76,11 +79,10 @@ class AsyncAgent:
|
||||
return total_rewards
|
||||
|
||||
def async_update(self, worker_network, optimizer):
|
||||
with self.network_lock:
|
||||
optimizer.zero_grad()
|
||||
for param, worker_param in zip(self.learning_network.parameters(), worker_network.parameters()):
|
||||
param._grad = worker_param.grad.clone().cpu()
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
for param, worker_param in zip(self.learning_network.parameters(), worker_network.parameters()):
|
||||
param._grad = worker_param.grad.clone().cpu()
|
||||
optimizer.step()
|
||||
|
||||
def worker(self, id):
|
||||
optimizer = self.optimizer_fn(self.learning_network.parameters())
|
||||
@@ -92,28 +94,32 @@ class AsyncAgent:
|
||||
episode = 0
|
||||
episode_steps = 0
|
||||
episode_return = 0
|
||||
episode_returns = [0]
|
||||
episode_returns = []
|
||||
update_target_network = False
|
||||
while True and not self.stop_signal.value:
|
||||
batch_states, batch_actions, batch_rewards = [], [], []
|
||||
if terminal:
|
||||
if id == 0:
|
||||
self.logger.info('episode %d, return %f, avg return %f, total steps %d' % (
|
||||
episode, episode_return, np.mean(episode_returns[-100: ]),
|
||||
if episode and id == 0:
|
||||
episode_returns.append(episode_return)
|
||||
self.logger.info('episode %d, return %f, avg return %f, episode steps %d, total steps %d' % (
|
||||
episode, episode_return, np.mean(episode_returns[-100: ]), episode_steps,
|
||||
self.total_steps.value))
|
||||
episode_steps = 0
|
||||
episode_returns.append(episode_return)
|
||||
episode_return = 0
|
||||
episode += 1
|
||||
terminal = False
|
||||
state = task.reset()
|
||||
buffer = [state] * self.history_length
|
||||
state = task.normalize_state(np.vstack(buffer))
|
||||
state = np.vstack(buffer)
|
||||
value = worker_network.predict(np.stack([state]))
|
||||
action = policy.sample(value.flatten())
|
||||
while not terminal and len(batch_states) < self.batch_size:
|
||||
episode_steps += 1
|
||||
with self.steps_lock:
|
||||
self.total_steps.value += 1
|
||||
self.total_steps.value += 1
|
||||
if self.total_steps.value % self.target_network_update_freq == 0:
|
||||
update_target_network = True
|
||||
batch_states.append(state)
|
||||
batch_actions.append(action)
|
||||
state, reward, terminal, _ = task.step(action)
|
||||
@@ -121,7 +127,7 @@ class AsyncAgent:
|
||||
episode_return += reward
|
||||
buffer.pop(0)
|
||||
buffer.append(state)
|
||||
state = task.normalize_state(np.vstack(buffer))
|
||||
state = np.vstack(buffer)
|
||||
value = worker_network.predict(np.stack([state]))
|
||||
action = policy.sample(value.flatten())
|
||||
policy.update_epsilon()
|
||||
@@ -139,27 +145,26 @@ class AsyncAgent:
|
||||
self.async_update(worker_network, optimizer)
|
||||
worker_network.load_state_dict(self.learning_network.state_dict())
|
||||
|
||||
if self.target_network_update_freq and \
|
||||
self.total_steps.value % self.target_network_update_freq == 0:
|
||||
if self.target_network is not None and update_target_network:
|
||||
with self.network_lock:
|
||||
self.target_network.load_state_dict(self.learning_network.state_dict())
|
||||
update_target_network = False
|
||||
|
||||
def save(self, file_name):
|
||||
with open(file_name, 'wb') as f:
|
||||
pickle.dump(self.learning_network.state_dict(), f)
|
||||
|
||||
def run(self):
|
||||
os.environ['OMP_NUM_THREADS'] = '1'
|
||||
procs = [mp.Process(target=self.worker, args=(i, )) for i in range(self.n_workers)]
|
||||
for p in procs: p.start()
|
||||
task = self.task_fn()
|
||||
test_network = self.network_fn()
|
||||
test_rewards = [0]
|
||||
test_points = [0]
|
||||
test_rewards = []
|
||||
test_points = []
|
||||
while True:
|
||||
steps = self.total_steps.value + 1
|
||||
if steps >= test_points[-1] + self.test_interval:
|
||||
test_points.append(steps)
|
||||
self.logger.info('Testing...')
|
||||
if steps % self.test_interval == 0:
|
||||
with self.network_lock:
|
||||
test_network.load_state_dict(self.learning_network.state_dict())
|
||||
self.save('data/%s-model-%s.bin' % (self.bootstrap_fn.__name__, task.name))
|
||||
@@ -169,6 +174,7 @@ class AsyncAgent:
|
||||
self.logger.info('total steps: %d, averaged return per episode: %f(%f)' %\
|
||||
(steps, np.mean(rewards), np.std(rewards) / np.sqrt(self.test_repetitions)))
|
||||
test_rewards.append(np.mean(rewards))
|
||||
test_points.append(steps)
|
||||
with open('data/%s-statistics-%s.bin' % (
|
||||
self.bootstrap_fn.__name__, task.name
|
||||
), 'wb') as f:
|
||||
|
||||
@@ -2,44 +2,6 @@ from async_agent import *
|
||||
from dqn_agent import *
|
||||
import logging
|
||||
|
||||
def async_cart_pole():
|
||||
config = dict()
|
||||
config['task_fn'] = lambda: CartPole()
|
||||
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001)
|
||||
config['network_fn'] = lambda gpu=True: FullyConnectedNet([8, 50, 200, 2], gpu=gpu)
|
||||
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=5000, min_epsilon=0.1)
|
||||
config['bootstrap_fn'] = OneStepQLearning
|
||||
# config['bootstrap_fn'] = NStepQLearning
|
||||
# config['bootstrap_fn'] = OneStepSarsa
|
||||
config['discount'] = 0.99
|
||||
config['target_network_update_freq'] = 200
|
||||
config['step_limit'] = 0
|
||||
config['n_workers'] = 8
|
||||
config['batch_size'] = 5
|
||||
config['test_interval'] = 4000
|
||||
config['test_repetitions'] = 50
|
||||
config['history_length'] = 2
|
||||
config['logger'] = gym.logger
|
||||
agent = AsyncAgent(**config)
|
||||
agent.run()
|
||||
|
||||
def async_lunar_lander():
|
||||
config = dict()
|
||||
config['task_fn'] = lambda: LunarLander()
|
||||
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001)
|
||||
config['network_fn'] = lambda gpu=True: FullyConnectedNet([8, 50, 200, 4], gpu=gpu)
|
||||
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=40000, min_epsilon=0.05)
|
||||
config['bootstrap_fn'] = OneStepQLearning
|
||||
config['discount'] = 0.99
|
||||
config['target_network_update_freq'] = 200
|
||||
config['step_limit'] = 5000
|
||||
config['n_workers'] = 8
|
||||
config['batch_size'] = 10
|
||||
config['test_interval'] = 1000
|
||||
config['test_repetitions'] = 5
|
||||
agent = AsyncAgent(**config)
|
||||
agent.run()
|
||||
|
||||
def dqn_cart_pole():
|
||||
config = dict()
|
||||
config['task_fn'] = lambda: CartPole()
|
||||
@@ -58,11 +20,32 @@ def dqn_cart_pole():
|
||||
agent = DQNAgent(**config)
|
||||
agent.run()
|
||||
|
||||
def async_cart_pole():
|
||||
config = dict()
|
||||
config['task_fn'] = lambda: CartPole()
|
||||
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001)
|
||||
config['network_fn'] = lambda: FullyConnectedNet([4, 50, 200, 2], gpu=False)
|
||||
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=5000, min_epsilon=0.1)
|
||||
config['bootstrap_fn'] = OneStepQLearning
|
||||
# config['bootstrap_fn'] = NStepQLearning
|
||||
# config['bootstrap_fn'] = OneStepSarsa
|
||||
config['discount'] = 0.99
|
||||
config['target_network_update_freq'] = 200
|
||||
config['step_limit'] = 0
|
||||
config['n_workers'] = 16
|
||||
config['batch_size'] = 6
|
||||
config['test_interval'] = 4000
|
||||
config['test_repetitions'] = 50
|
||||
config['history_length'] = 1
|
||||
config['logger'] = gym.logger
|
||||
agent = AsyncAgent(**config)
|
||||
agent.run()
|
||||
|
||||
def a3c_cart_pole():
|
||||
config = dict()
|
||||
config['task_fn'] = lambda: CartPole()
|
||||
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001)
|
||||
config['network_fn'] = lambda gpu=True: FCActorCriticNet([8, 200, 2], gpu=gpu)
|
||||
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001)
|
||||
config['network_fn'] = lambda: FCActorCriticNet([4, 200, 2], gpu=False)
|
||||
config['policy_fn'] = SamplePolicy
|
||||
config['bootstrap_fn'] = AdvantageActorCritic
|
||||
config['discount'] = 0.99
|
||||
@@ -71,7 +54,7 @@ def a3c_cart_pole():
|
||||
config['n_workers'] = 16
|
||||
config['batch_size'] = 6
|
||||
config['test_interval'] = 4000
|
||||
config['history_length'] = 2
|
||||
config['history_length'] = 1
|
||||
config['test_repetitions'] = 50
|
||||
config['logger'] = gym.logger
|
||||
agent = AsyncAgent(**config)
|
||||
@@ -81,7 +64,7 @@ def dqn_pixel_atari(name):
|
||||
config = dict()
|
||||
history_length = 4
|
||||
n_actions = 6
|
||||
config['task_fn'] = lambda: PixelAtari(name, 30, 4)
|
||||
config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, normalized_state=False)
|
||||
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01)
|
||||
config['network_fn'] = lambda optimizer_fn: ConvNet(history_length, n_actions, optimizer_fn)
|
||||
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
|
||||
@@ -99,22 +82,22 @@ def dqn_pixel_atari(name):
|
||||
|
||||
def async_pixel_atari(name):
|
||||
config = dict()
|
||||
history_length = 4
|
||||
history_length = 1
|
||||
n_actions = 6
|
||||
config['task_fn'] = lambda: PixelAtari(name, 30, 4)
|
||||
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01)
|
||||
config['network_fn'] = lambda : ConvNet(history_length, n_actions, gpu=True)
|
||||
config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4)
|
||||
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001)
|
||||
config['network_fn'] = lambda: ConvNet(history_length, n_actions, gpu=False)
|
||||
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
|
||||
config['bootstrap_fn'] = OneStepQLearning
|
||||
# config['bootstrap_fn'] = NStepQLearning
|
||||
# config['bootstrap_fn'] = OneStepSarsa
|
||||
config['discount'] = 0.99
|
||||
config['target_network_update_freq'] = 10000
|
||||
config['step_limit'] = 0
|
||||
config['n_workers'] = 1
|
||||
config['batch_size'] = 32
|
||||
config['step_limit'] = 10000
|
||||
config['n_workers'] = 16
|
||||
config['batch_size'] = 20
|
||||
config['test_interval'] = 50000
|
||||
config['test_repetitions'] = 50
|
||||
config['test_repetitions'] = 1
|
||||
config['history_length'] = history_length
|
||||
config['logger'] = gym.logger
|
||||
agent = AsyncAgent(**config)
|
||||
@@ -122,21 +105,21 @@ def async_pixel_atari(name):
|
||||
|
||||
def a3c_pixel_atari(name):
|
||||
config = dict()
|
||||
history_length = 4
|
||||
history_length = 1
|
||||
n_actions = 6
|
||||
config['task_fn'] = lambda: PixelAtari(name, 30, 4)
|
||||
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.0001, alpha=0.99, eps=0.01)
|
||||
config['network_fn'] = lambda gpu=True: ConvActorCriticNet(history_length, n_actions, gpu=gpu)
|
||||
config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4)
|
||||
config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001)
|
||||
config['network_fn'] = lambda: ConvActorCriticNet(history_length, n_actions, gpu=False)
|
||||
config['policy_fn'] = SamplePolicy
|
||||
config['bootstrap_fn'] = AdvantageActorCritic
|
||||
config['discount'] = 0.99
|
||||
config['target_network_update_freq'] = 0
|
||||
config['step_limit'] = 0
|
||||
config['step_limit'] = 10000
|
||||
config['n_workers'] = 16
|
||||
config['batch_size'] = 10
|
||||
config['test_interval'] = 10000
|
||||
config['test_repetitions'] = 20
|
||||
config['history_length'] = 4
|
||||
config['batch_size'] = 20
|
||||
config['test_interval'] = 50000
|
||||
config['test_repetitions'] = 1
|
||||
config['history_length'] = history_length
|
||||
config['logger'] = gym.logger
|
||||
agent = AsyncAgent(**config)
|
||||
agent.run()
|
||||
@@ -144,12 +127,12 @@ def a3c_pixel_atari(name):
|
||||
if __name__ == '__main__':
|
||||
# gym.logger.setLevel(logging.DEBUG)
|
||||
gym.logger.setLevel(logging.INFO)
|
||||
benchmark = gym.benchmark_spec('Atari40M')
|
||||
|
||||
# async_cart_pole()
|
||||
# a3c_cart_pole()
|
||||
# async_lunar_lander()
|
||||
async_pixel_atari('PongNoFrameskip-v3')
|
||||
# dqn_cart_pole()
|
||||
# dqn_pixel_atari('BreakoutNoFrameskip-v3')
|
||||
# async_pixel_atari('BreakoutNoFrameskip-v3')
|
||||
a3c_pixel_atari('BreakoutNoFrameskip-v3')
|
||||
# a3c_pixel_atari('BreakoutNoFrameskip-v3')
|
||||
# a3c_cart_pole()
|
||||
# a3c_pixel_atari('PongNoFrameskip-v3')
|
||||
|
||||
+14
-7
@@ -10,6 +10,7 @@ import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import numpy as np
|
||||
|
||||
# Base class for all kinds of network
|
||||
class BasicNet:
|
||||
def __init__(self, optimizer_fn, gpu):
|
||||
if optimizer_fn is not None:
|
||||
@@ -26,6 +27,7 @@ class BasicNet:
|
||||
x = x.cuda()
|
||||
return Variable(x)
|
||||
|
||||
# Base class for value based methods
|
||||
class VanillaNet(BasicNet):
|
||||
def predict(self, x, to_numpy=True):
|
||||
y = self.forward(x)
|
||||
@@ -39,6 +41,7 @@ class VanillaNet(BasicNet):
|
||||
loss = self.criterion(y, targets)
|
||||
loss.backward()
|
||||
|
||||
# Base class for actor critic method
|
||||
class ActorCriticNet(BasicNet):
|
||||
def predict(self, x):
|
||||
phi = self.forward(x)
|
||||
@@ -53,16 +56,17 @@ class ActorCriticNet(BasicNet):
|
||||
log_prob = log_prob_.gather(1, actions)
|
||||
advantage = (rewards - state_value).detach()
|
||||
policy_loss = -torch.sum(log_prob * advantage)
|
||||
value_loss = 0.5 * torch.sum(torch.pow(state_value - rewards, 2))
|
||||
value_loss = 0.5 * torch.sum(torch.pow(rewards - state_value, 2))
|
||||
entropy = -torch.sum(torch.mul(prob, log_prob_))
|
||||
(policy_loss + value_loss - self.xentropy_weight * entropy).backward()
|
||||
loss = policy_loss + value_loss - self.xentropy_weight * entropy
|
||||
loss.backward()
|
||||
nn.utils.clip_grad_norm(self.parameters(), self.grad_threshold)
|
||||
|
||||
def critic(self, x):
|
||||
phi = self.forward(x)
|
||||
return self.fc_critic(phi).cpu().data.numpy()
|
||||
|
||||
|
||||
# Network for CartPole with value based methods
|
||||
class FullyConnectedNet(nn.Module, VanillaNet):
|
||||
def __init__(self, dims, optimizer_fn=None, gpu=True):
|
||||
super(FullyConnectedNet, self).__init__()
|
||||
@@ -80,6 +84,7 @@ class FullyConnectedNet(nn.Module, VanillaNet):
|
||||
y = self.fc3(y)
|
||||
return y
|
||||
|
||||
# Network for pixel Atari game with value based methods
|
||||
class ConvNet(nn.Module, VanillaNet):
|
||||
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
|
||||
super(ConvNet, self).__init__()
|
||||
@@ -100,6 +105,7 @@ class ConvNet(nn.Module, VanillaNet):
|
||||
y = F.relu(self.fc4(y))
|
||||
return self.fc5(y)
|
||||
|
||||
# Network for CartPole with actor critic
|
||||
class FCActorCriticNet(nn.Module, ActorCriticNet):
|
||||
def __init__(self,
|
||||
dims,
|
||||
@@ -120,6 +126,7 @@ class FCActorCriticNet(nn.Module, ActorCriticNet):
|
||||
phi = self.fc1(x)
|
||||
return phi
|
||||
|
||||
# Network for pixel Atari game with actor critic
|
||||
class ConvActorCriticNet(nn.Module, ActorCriticNet):
|
||||
def __init__(self,
|
||||
in_channels,
|
||||
@@ -140,9 +147,9 @@ class ConvActorCriticNet(nn.Module, ActorCriticNet):
|
||||
|
||||
def forward(self, x):
|
||||
x = self.to_torch_variable(x)
|
||||
y = F.relu(self.conv1(x))
|
||||
y = F.relu(self.conv2(y))
|
||||
y = F.relu(self.conv3(y))
|
||||
y = F.elu(self.conv1(x))
|
||||
y = F.elu(self.conv2(y))
|
||||
y = F.elu(self.conv3(y))
|
||||
y = y.view(y.size(0), -1)
|
||||
return F.relu(self.fc4(y))
|
||||
return F.elu(self.fc4(y))
|
||||
|
||||
|
||||
@@ -9,19 +9,22 @@ import numpy as np
|
||||
from atari_wrapper import *
|
||||
|
||||
class BasicTask:
|
||||
def transfer_state(self, state):
|
||||
return state
|
||||
def __init__(self):
|
||||
self.normalized_state = True
|
||||
|
||||
def normalize_state(self, state):
|
||||
return state
|
||||
|
||||
def reset(self):
|
||||
state = self.env.reset()
|
||||
return self.transfer_state(state)
|
||||
if self.normalized_state:
|
||||
return self.normalize_state(state)
|
||||
return state
|
||||
|
||||
def step(self, action):
|
||||
next_state, reward, done, info = self.env.step(action)
|
||||
next_state = self.transfer_state(next_state)
|
||||
if self.normalized_state:
|
||||
next_state = self.normalize_state(next_state)
|
||||
return next_state, np.sign(reward), done, info
|
||||
|
||||
class MountainCar(BasicTask):
|
||||
@@ -29,6 +32,7 @@ class MountainCar(BasicTask):
|
||||
success_threshold = -110
|
||||
|
||||
def __init__(self):
|
||||
BasicTask.__init__(self)
|
||||
self.env = gym.make(self.name)
|
||||
self.env._max_episode_steps = sys.maxsize
|
||||
|
||||
@@ -37,6 +41,7 @@ class CartPole(BasicTask):
|
||||
success_threshold = 195
|
||||
|
||||
def __init__(self):
|
||||
BasicTask.__init__(self)
|
||||
self.env = gym.make(self.name)
|
||||
|
||||
class LunarLander(BasicTask):
|
||||
@@ -44,12 +49,15 @@ class LunarLander(BasicTask):
|
||||
success_threshold = 200
|
||||
|
||||
def __init__(self):
|
||||
BasicTask.__init__(self)
|
||||
self.env = gym.make(self.name)
|
||||
|
||||
class PixelAtari(BasicTask):
|
||||
success_threshold = 1000
|
||||
|
||||
def __init__(self, name, no_op, frame_skip):
|
||||
def __init__(self, name, no_op, frame_skip, normalized_state=True):
|
||||
BasicTask.__init__(self)
|
||||
self.normalized_state = normalized_state
|
||||
self.name = name
|
||||
env = gym.make(name)
|
||||
assert 'NoFrameskip' in env.spec.id
|
||||
|
||||
Reference in New Issue
Block a user