diff --git a/async_agent.py b/async_agent.py index fef1eb4..09b7a34 100644 --- a/async_agent.py +++ b/async_agent.py @@ -12,6 +12,7 @@ from task import * from network import * from bootstrap import * import pickle +import os class AsyncAgent: def __init__(self, @@ -30,12 +31,14 @@ class AsyncAgent: history_length, logger): self.network_fn = network_fn - self.learning_network = network_fn(False) + self.learning_network = network_fn() self.learning_network.share_memory() if bootstrap_fn != AdvantageActorCritic: - self.target_network = network_fn(False) + self.target_network = network_fn() self.target_network.share_memory() self.target_network.load_state_dict(self.learning_network.state_dict()) + else: + self.target_network = None self.bootstrap_fn = bootstrap_fn self.optimizer_fn = optimizer_fn @@ -63,7 +66,7 @@ class AsyncAgent: terminal = False buffer = [state] * self.history_length while not terminal and (not self.step_limit or steps < self.step_limit): - state = task.normalize_state(np.vstack(buffer)) + state = np.vstack(buffer) action_values = network.predict(np.stack([state])) steps += 1 action = np.argmax(action_values.flatten()) @@ -76,11 +79,10 @@ class AsyncAgent: return total_rewards def async_update(self, worker_network, optimizer): - with self.network_lock: - optimizer.zero_grad() - for param, worker_param in zip(self.learning_network.parameters(), worker_network.parameters()): - param._grad = worker_param.grad.clone().cpu() - optimizer.step() + optimizer.zero_grad() + for param, worker_param in zip(self.learning_network.parameters(), worker_network.parameters()): + param._grad = worker_param.grad.clone().cpu() + optimizer.step() def worker(self, id): optimizer = self.optimizer_fn(self.learning_network.parameters()) @@ -92,28 +94,32 @@ class AsyncAgent: episode = 0 episode_steps = 0 episode_return = 0 - episode_returns = [0] + episode_returns = [] + update_target_network = False while True and not self.stop_signal.value: batch_states, batch_actions, batch_rewards = [], [], [] if terminal: - if id == 0: - self.logger.info('episode %d, return %f, avg return %f, total steps %d' % ( - episode, episode_return, np.mean(episode_returns[-100: ]), + if episode and id == 0: + episode_returns.append(episode_return) + self.logger.info('episode %d, return %f, avg return %f, episode steps %d, total steps %d' % ( + episode, episode_return, np.mean(episode_returns[-100: ]), episode_steps, self.total_steps.value)) episode_steps = 0 - episode_returns.append(episode_return) episode_return = 0 episode += 1 terminal = False state = task.reset() buffer = [state] * self.history_length - state = task.normalize_state(np.vstack(buffer)) + state = np.vstack(buffer) value = worker_network.predict(np.stack([state])) action = policy.sample(value.flatten()) while not terminal and len(batch_states) < self.batch_size: episode_steps += 1 with self.steps_lock: self.total_steps.value += 1 + self.total_steps.value += 1 + if self.total_steps.value % self.target_network_update_freq == 0: + update_target_network = True batch_states.append(state) batch_actions.append(action) state, reward, terminal, _ = task.step(action) @@ -121,7 +127,7 @@ class AsyncAgent: episode_return += reward buffer.pop(0) buffer.append(state) - state = task.normalize_state(np.vstack(buffer)) + state = np.vstack(buffer) value = worker_network.predict(np.stack([state])) action = policy.sample(value.flatten()) policy.update_epsilon() @@ -139,27 +145,26 @@ class AsyncAgent: self.async_update(worker_network, optimizer) worker_network.load_state_dict(self.learning_network.state_dict()) - if self.target_network_update_freq and \ - self.total_steps.value % self.target_network_update_freq == 0: + if self.target_network is not None and update_target_network: with self.network_lock: self.target_network.load_state_dict(self.learning_network.state_dict()) + update_target_network = False def save(self, file_name): with open(file_name, 'wb') as f: pickle.dump(self.learning_network.state_dict(), f) def run(self): + os.environ['OMP_NUM_THREADS'] = '1' procs = [mp.Process(target=self.worker, args=(i, )) for i in range(self.n_workers)] for p in procs: p.start() task = self.task_fn() test_network = self.network_fn() - test_rewards = [0] - test_points = [0] + test_rewards = [] + test_points = [] while True: steps = self.total_steps.value + 1 - if steps >= test_points[-1] + self.test_interval: - test_points.append(steps) - self.logger.info('Testing...') + if steps % self.test_interval == 0: with self.network_lock: test_network.load_state_dict(self.learning_network.state_dict()) self.save('data/%s-model-%s.bin' % (self.bootstrap_fn.__name__, task.name)) @@ -169,6 +174,7 @@ class AsyncAgent: self.logger.info('total steps: %d, averaged return per episode: %f(%f)' %\ (steps, np.mean(rewards), np.std(rewards) / np.sqrt(self.test_repetitions))) test_rewards.append(np.mean(rewards)) + test_points.append(steps) with open('data/%s-statistics-%s.bin' % ( self.bootstrap_fn.__name__, task.name ), 'wb') as f: diff --git a/main.py b/main.py index eb9b7a8..7a441ad 100644 --- a/main.py +++ b/main.py @@ -2,44 +2,6 @@ from async_agent import * from dqn_agent import * import logging -def async_cart_pole(): - config = dict() - config['task_fn'] = lambda: CartPole() - config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001) - config['network_fn'] = lambda gpu=True: FullyConnectedNet([8, 50, 200, 2], gpu=gpu) - config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=5000, min_epsilon=0.1) - config['bootstrap_fn'] = OneStepQLearning - # config['bootstrap_fn'] = NStepQLearning - # config['bootstrap_fn'] = OneStepSarsa - config['discount'] = 0.99 - config['target_network_update_freq'] = 200 - config['step_limit'] = 0 - config['n_workers'] = 8 - config['batch_size'] = 5 - config['test_interval'] = 4000 - config['test_repetitions'] = 50 - config['history_length'] = 2 - config['logger'] = gym.logger - agent = AsyncAgent(**config) - agent.run() - -def async_lunar_lander(): - config = dict() - config['task_fn'] = lambda: LunarLander() - config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001) - config['network_fn'] = lambda gpu=True: FullyConnectedNet([8, 50, 200, 4], gpu=gpu) - config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=40000, min_epsilon=0.05) - config['bootstrap_fn'] = OneStepQLearning - config['discount'] = 0.99 - config['target_network_update_freq'] = 200 - config['step_limit'] = 5000 - config['n_workers'] = 8 - config['batch_size'] = 10 - config['test_interval'] = 1000 - config['test_repetitions'] = 5 - agent = AsyncAgent(**config) - agent.run() - def dqn_cart_pole(): config = dict() config['task_fn'] = lambda: CartPole() @@ -58,11 +20,32 @@ def dqn_cart_pole(): agent = DQNAgent(**config) agent.run() +def async_cart_pole(): + config = dict() + config['task_fn'] = lambda: CartPole() + config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001) + config['network_fn'] = lambda: FullyConnectedNet([4, 50, 200, 2], gpu=False) + config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=5000, min_epsilon=0.1) + config['bootstrap_fn'] = OneStepQLearning + # config['bootstrap_fn'] = NStepQLearning + # config['bootstrap_fn'] = OneStepSarsa + config['discount'] = 0.99 + config['target_network_update_freq'] = 200 + config['step_limit'] = 0 + config['n_workers'] = 16 + config['batch_size'] = 6 + config['test_interval'] = 4000 + config['test_repetitions'] = 50 + config['history_length'] = 1 + config['logger'] = gym.logger + agent = AsyncAgent(**config) + agent.run() + def a3c_cart_pole(): config = dict() config['task_fn'] = lambda: CartPole() - config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001) - config['network_fn'] = lambda gpu=True: FCActorCriticNet([8, 200, 2], gpu=gpu) + config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001) + config['network_fn'] = lambda: FCActorCriticNet([4, 200, 2], gpu=False) config['policy_fn'] = SamplePolicy config['bootstrap_fn'] = AdvantageActorCritic config['discount'] = 0.99 @@ -71,7 +54,7 @@ def a3c_cart_pole(): config['n_workers'] = 16 config['batch_size'] = 6 config['test_interval'] = 4000 - config['history_length'] = 2 + config['history_length'] = 1 config['test_repetitions'] = 50 config['logger'] = gym.logger agent = AsyncAgent(**config) @@ -81,7 +64,7 @@ def dqn_pixel_atari(name): config = dict() history_length = 4 n_actions = 6 - config['task_fn'] = lambda: PixelAtari(name, 30, 4) + config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, normalized_state=False) config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) config['network_fn'] = lambda optimizer_fn: ConvNet(history_length, n_actions, optimizer_fn) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) @@ -99,22 +82,22 @@ def dqn_pixel_atari(name): def async_pixel_atari(name): config = dict() - history_length = 4 + history_length = 1 n_actions = 6 - config['task_fn'] = lambda: PixelAtari(name, 30, 4) - config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) - config['network_fn'] = lambda : ConvNet(history_length, n_actions, gpu=True) + config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4) + config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001) + config['network_fn'] = lambda: ConvNet(history_length, n_actions, gpu=False) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) config['bootstrap_fn'] = OneStepQLearning # config['bootstrap_fn'] = NStepQLearning # config['bootstrap_fn'] = OneStepSarsa config['discount'] = 0.99 config['target_network_update_freq'] = 10000 - config['step_limit'] = 0 - config['n_workers'] = 1 - config['batch_size'] = 32 + config['step_limit'] = 10000 + config['n_workers'] = 16 + config['batch_size'] = 20 config['test_interval'] = 50000 - config['test_repetitions'] = 50 + config['test_repetitions'] = 1 config['history_length'] = history_length config['logger'] = gym.logger agent = AsyncAgent(**config) @@ -122,21 +105,21 @@ def async_pixel_atari(name): def a3c_pixel_atari(name): config = dict() - history_length = 4 + history_length = 1 n_actions = 6 - config['task_fn'] = lambda: PixelAtari(name, 30, 4) - config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.0001, alpha=0.99, eps=0.01) - config['network_fn'] = lambda gpu=True: ConvActorCriticNet(history_length, n_actions, gpu=gpu) + config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4) + config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001) + config['network_fn'] = lambda: ConvActorCriticNet(history_length, n_actions, gpu=False) config['policy_fn'] = SamplePolicy config['bootstrap_fn'] = AdvantageActorCritic config['discount'] = 0.99 config['target_network_update_freq'] = 0 - config['step_limit'] = 0 + config['step_limit'] = 10000 config['n_workers'] = 16 - config['batch_size'] = 10 - config['test_interval'] = 10000 - config['test_repetitions'] = 20 - config['history_length'] = 4 + config['batch_size'] = 20 + config['test_interval'] = 50000 + config['test_repetitions'] = 1 + config['history_length'] = history_length config['logger'] = gym.logger agent = AsyncAgent(**config) agent.run() @@ -144,12 +127,12 @@ def a3c_pixel_atari(name): if __name__ == '__main__': # gym.logger.setLevel(logging.DEBUG) gym.logger.setLevel(logging.INFO) - benchmark = gym.benchmark_spec('Atari40M') # async_cart_pole() - # a3c_cart_pole() - # async_lunar_lander() + async_pixel_atari('PongNoFrameskip-v3') # dqn_cart_pole() # dqn_pixel_atari('BreakoutNoFrameskip-v3') # async_pixel_atari('BreakoutNoFrameskip-v3') - a3c_pixel_atari('BreakoutNoFrameskip-v3') + # a3c_pixel_atari('BreakoutNoFrameskip-v3') + # a3c_cart_pole() + # a3c_pixel_atari('PongNoFrameskip-v3') diff --git a/network.py b/network.py index 50103cf..45fdfff 100644 --- a/network.py +++ b/network.py @@ -10,6 +10,7 @@ import torch.nn as nn import torch.nn.functional as F import numpy as np +# Base class for all kinds of network class BasicNet: def __init__(self, optimizer_fn, gpu): if optimizer_fn is not None: @@ -26,6 +27,7 @@ class BasicNet: x = x.cuda() return Variable(x) +# Base class for value based methods class VanillaNet(BasicNet): def predict(self, x, to_numpy=True): y = self.forward(x) @@ -39,6 +41,7 @@ class VanillaNet(BasicNet): loss = self.criterion(y, targets) loss.backward() +# Base class for actor critic method class ActorCriticNet(BasicNet): def predict(self, x): phi = self.forward(x) @@ -53,16 +56,17 @@ class ActorCriticNet(BasicNet): log_prob = log_prob_.gather(1, actions) advantage = (rewards - state_value).detach() policy_loss = -torch.sum(log_prob * advantage) - value_loss = 0.5 * torch.sum(torch.pow(state_value - rewards, 2)) + value_loss = 0.5 * torch.sum(torch.pow(rewards - state_value, 2)) entropy = -torch.sum(torch.mul(prob, log_prob_)) - (policy_loss + value_loss - self.xentropy_weight * entropy).backward() + loss = policy_loss + value_loss - self.xentropy_weight * entropy + loss.backward() nn.utils.clip_grad_norm(self.parameters(), self.grad_threshold) def critic(self, x): phi = self.forward(x) return self.fc_critic(phi).cpu().data.numpy() - +# Network for CartPole with value based methods class FullyConnectedNet(nn.Module, VanillaNet): def __init__(self, dims, optimizer_fn=None, gpu=True): super(FullyConnectedNet, self).__init__() @@ -80,6 +84,7 @@ class FullyConnectedNet(nn.Module, VanillaNet): y = self.fc3(y) return y +# Network for pixel Atari game with value based methods class ConvNet(nn.Module, VanillaNet): def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True): super(ConvNet, self).__init__() @@ -100,6 +105,7 @@ class ConvNet(nn.Module, VanillaNet): y = F.relu(self.fc4(y)) return self.fc5(y) +# Network for CartPole with actor critic class FCActorCriticNet(nn.Module, ActorCriticNet): def __init__(self, dims, @@ -120,6 +126,7 @@ class FCActorCriticNet(nn.Module, ActorCriticNet): phi = self.fc1(x) return phi +# Network for pixel Atari game with actor critic class ConvActorCriticNet(nn.Module, ActorCriticNet): def __init__(self, in_channels, @@ -140,9 +147,9 @@ class ConvActorCriticNet(nn.Module, ActorCriticNet): def forward(self, x): x = self.to_torch_variable(x) - y = F.relu(self.conv1(x)) - y = F.relu(self.conv2(y)) - y = F.relu(self.conv3(y)) + y = F.elu(self.conv1(x)) + y = F.elu(self.conv2(y)) + y = F.elu(self.conv3(y)) y = y.view(y.size(0), -1) - return F.relu(self.fc4(y)) + return F.elu(self.fc4(y)) diff --git a/task.py b/task.py index dd45eb8..7026c23 100644 --- a/task.py +++ b/task.py @@ -9,19 +9,22 @@ import numpy as np from atari_wrapper import * class BasicTask: - def transfer_state(self, state): - return state + def __init__(self): + self.normalized_state = True def normalize_state(self, state): return state def reset(self): state = self.env.reset() - return self.transfer_state(state) + if self.normalized_state: + return self.normalize_state(state) + return state def step(self, action): next_state, reward, done, info = self.env.step(action) - next_state = self.transfer_state(next_state) + if self.normalized_state: + next_state = self.normalize_state(next_state) return next_state, np.sign(reward), done, info class MountainCar(BasicTask): @@ -29,6 +32,7 @@ class MountainCar(BasicTask): success_threshold = -110 def __init__(self): + BasicTask.__init__(self) self.env = gym.make(self.name) self.env._max_episode_steps = sys.maxsize @@ -37,6 +41,7 @@ class CartPole(BasicTask): success_threshold = 195 def __init__(self): + BasicTask.__init__(self) self.env = gym.make(self.name) class LunarLander(BasicTask): @@ -44,12 +49,15 @@ class LunarLander(BasicTask): success_threshold = 200 def __init__(self): + BasicTask.__init__(self) self.env = gym.make(self.name) class PixelAtari(BasicTask): success_threshold = 1000 - def __init__(self, name, no_op, frame_skip): + def __init__(self, name, no_op, frame_skip, normalized_state=True): + BasicTask.__init__(self) + self.normalized_state = normalized_state self.name = name env = gym.make(name) assert 'NoFrameskip' in env.spec.id