diff --git a/async_agent.py b/async_agent.py index 6e4d346..f29bf4d 100644 --- a/async_agent.py +++ b/async_agent.py @@ -59,6 +59,7 @@ class AsyncAgent: self.test_repetitions = test_repetitions self.logger = logger self.history_length = history_length + self.tag = '' def deterministic_episode(self, task, network): state = task.reset() @@ -73,12 +74,14 @@ class AsyncAgent: total_rewards += reward if terminal: break + bootstrap.reset() return total_rewards def worker(self, id): optimizer = self.optimizer_fn(self.learning_network.parameters()) worker_network = self.network_fn() worker_network.load_state_dict(self.learning_network.state_dict()) + bootstrap = self.bootstrap(self) task = self.task_fn() policy = self.policy_fn() @@ -125,8 +128,7 @@ class AsyncAgent: state = next_state if self.target_network and self.total_steps.value % self.target_network_update_freq == 0: - with self.network_lock: - self.target_network.load_state_dict(self.learning_network.state_dict()) + self.target_network.load_state_dict(self.learning_network.state_dict()) def save(self, file_name): with open(file_name, 'wb') as f: @@ -142,9 +144,8 @@ class AsyncAgent: while True: steps = self.total_steps.value + 1 if steps % self.test_interval == 0: - with self.network_lock: - test_network.load_state_dict(self.learning_network.state_dict()) - self.save('data/%s-model-%s.bin' % (self.bootstrap.__name__, self.task.name)) + test_network.load_state_dict(self.learning_network.state_dict()) + self.save('data/%s%s-model-%s.bin' % (self.tag, self.bootstrap.__name__, self.task.name)) rewards = np.zeros(self.test_repetitions) for i in range(self.test_repetitions): rewards[i] = self.deterministic_episode(self.task, test_network) @@ -152,8 +153,8 @@ class AsyncAgent: (steps, np.mean(rewards), np.std(rewards) / np.sqrt(self.test_repetitions))) test_rewards.append(np.mean(rewards)) test_points.append(steps) - with open('data/%s-statistics-%s.bin' % ( - self.bootstrap.__name__, self.task.name + with open('data/%s%s-statistics-%s.bin' % ( + self.tag, self.bootstrap.__name__, self.task.name ), 'wb') as f: pickle.dump([test_points, test_rewards], f) if np.mean(rewards) > self.task.success_threshold: diff --git a/atari_wrapper.py b/atari_wrapper.py index 7f9ef10..936eec0 100644 --- a/atari_wrapper.py +++ b/atari_wrapper.py @@ -134,7 +134,7 @@ class ProcessFrame(gym.Wrapper): elif frame_size == 42: self.process_fn = _process_frame42 else: - assert(False, "Unknown frame size") + assert False, "Unknown frame size" def _step(self, action): obs, reward, done, info = self.env.step(action) diff --git a/bootstrap.py b/bootstrap.py index 0a730a3..ab99240 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -10,6 +10,9 @@ from torch.autograd import Variable class OneStepSarsa: def __init__(self, agent): self.agent = agent + self.reset() + + def reset(self): self.pending = [] def process_state(self, network, state): @@ -36,13 +39,16 @@ class OneStepSarsa: q_next = self.agent.discount * q_next + reward q = q.gather(1, Variable(torch.LongTensor([[action]]))) loss += 0.5 * (q - Variable(q_next)).pow(2) - self.pending = [] + self.reset() return loss class OneStepQLearning: def __init__(self, agent): self.agent = agent + self.reset() + + def reset(self): self.pending = [] def process_state(self, network, state): @@ -63,12 +69,15 @@ class OneStepQLearning: q_next = self.agent.discount * q_next + reward q = q.gather(1, Variable(torch.LongTensor([[action]]))) loss += 0.5 * (q - Variable(q_next)).pow(2) - self.pending = [] + self.reset() return loss class NStepQLearning: def __init__(self, agent): self.agent = agent + self.reset() + + def reset(self): self.pending = [] def process_state(self, network, state): @@ -92,12 +101,15 @@ class NStepQLearning: q, action, reward = self.pending[i] R = reward + self.agent.discount * R loss += 0.5 * (Variable(R) - q.gather(1, Variable(torch.LongTensor([[action]])))).pow(2) - self.pending = [] + self.reset() return loss class AdvantageActorCritic: def __init__(self, agent): self.agent = agent + self.reset() + + def reset(self): self.pending = [] def process_state(self, network, state): @@ -122,6 +134,6 @@ class AdvantageActorCritic: loss += 0.5 * advantage.pow(2) loss += -log_prob.gather(1, Variable(torch.LongTensor([[action]]))) * Variable(advantage.data) loss += 0.01 * torch.sum(torch.mul(prob, log_prob)) - self.pending = [] + self.reset() return loss diff --git a/images/A3C-PongNoFrameskip-v3.png b/images/A3C-PongNoFrameskip-v3.png deleted file mode 100644 index 63b5add..0000000 Binary files a/images/A3C-PongNoFrameskip-v3.png and /dev/null differ diff --git a/main.py b/main.py index d04ce61..e62d4ec 100644 --- a/main.py +++ b/main.py @@ -6,8 +6,8 @@ def dqn_cart_pole(): config = dict() config['task_fn'] = lambda: CartPole() config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, 0.001) - config['network_fn'] = lambda optimizer_fn: FullyConnectedNet([8, 50, 200, 2], optimizer_fn) - # config['network_fn'] = lambda optimizer_fn: DuelingFullyConnectedNet([8, 50, 200, 2], optimizer_fn) + config['network_fn'] = lambda optimizer_fn: FCNet([8, 50, 200, 2], optimizer_fn) + # config['network_fn'] = lambda optimizer_fn: DuelingFCNet([8, 50, 200, 2], optimizer_fn) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1) config['replay_fn'] = lambda: Replay(memory_size=10000, batch_size=10) config['discount'] = 0.99 @@ -27,7 +27,7 @@ def async_cart_pole(): config = dict() config['task_fn'] = lambda: CartPole() config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001) - config['network_fn'] = lambda: FullyConnectedNet([4, 50, 200, 2]) + config['network_fn'] = lambda: FCNet([4, 50, 200, 2]) config['policy_fn'] = lambda: GreedyPolicy(epsilon=0.5, final_step=5000, min_epsilon=0.1) config['bootstrap'] = OneStepQLearning # config['bootstrap'] = NStepQLearning @@ -49,7 +49,7 @@ def a3c_cart_pole(): config = dict() config['task_fn'] = lambda: CartPole() config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001) - config['network_fn'] = lambda: FCActorCriticNet([4, 200, 2]) + config['network_fn'] = lambda: ActorCriticFCNet([4, 200, 2]) config['policy_fn'] = SamplePolicy config['bootstrap'] = AdvantageActorCritic config['discount'] = 0.99 @@ -70,8 +70,8 @@ def dqn_pixel_atari(name): n_actions = 6 config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, normalized_state=False) config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) - # config['network_fn'] = lambda optimizer_fn: ConvNet(history_length, n_actions, optimizer_fn) - config['network_fn'] = lambda optimizer_fn: DuelingConvNet(history_length, n_actions, optimizer_fn) + config['network_fn'] = lambda optimizer_fn: NatureConvNet(history_length, n_actions, optimizer_fn) + # config['network_fn'] = lambda optimizer_fn: DuelingNatureConvNet(history_length, n_actions, optimizer_fn) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) config['replay_fn'] = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8) config['discount'] = 0.99 @@ -95,20 +95,19 @@ def async_pixel_atari(name): config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, frame_size=42) config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001) config['network_fn'] = lambda: OpenAIConvNet(history_length, - n_actions, - LSTM=False) - config['policy_fn'] = lambda: StochasticGreedyPolicy(epsilons=[1.0, 1.0, 1.0], - final_step=1000000, - min_epsilons=[0.1, 0.01, 0.5], + n_actions) + config['policy_fn'] = lambda: StochasticGreedyPolicy(epsilons=[0.5, 0.5, 0.5], + final_step=2000000, + min_epsilons=[0.1, 0.01, 0.2], probs=[0.4, 0.3, 0.3]) # config['bootstrap'] = OneStepQLearning - config['bootstrap'] = NStepQLearning - # config['bootstrap'] = OneStepSarsa + # config['bootstrap'] = NStepQLearning + config['bootstrap'] = OneStepSarsa config['discount'] = 0.99 config['target_network_update_freq'] = 10000 config['step_limit'] = 10000 config['n_workers'] = 16 - config['update_interval'] = 32 + config['update_interval'] = 20 config['test_interval'] = 50000 config['test_repetitions'] = 1 config['history_length'] = history_length @@ -122,9 +121,9 @@ def a3c_pixel_atari(name): n_actions = 6 config['task_fn'] = lambda: PixelAtari(name, no_op=30, frame_skip=4, frame_size=42) config['optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=0.0001) - config['network_fn'] = lambda: OpenAIConvActorCriticNet(history_length, + config['network_fn'] = lambda: OpenAIActorCriticConvNet(history_length, n_actions, - LSTM=True) + LSTM=False) config['policy_fn'] = SamplePolicy config['bootstrap'] = AdvantageActorCritic config['discount'] = 0.99 @@ -137,6 +136,7 @@ def a3c_pixel_atari(name): config['history_length'] = history_length config['logger'] = gym.logger agent = AsyncAgent(**config) + agent.tag = '' agent.run() if __name__ == '__main__': @@ -148,9 +148,9 @@ if __name__ == '__main__': # a3c_cart_pole() # dqn_pixel_atari('PongNoFrameskip-v3') - # async_pixel_atari('PongNoFrameskip-v3') + async_pixel_atari('PongNoFrameskip-v3') # a3c_pixel_atari('PongNoFrameskip-v3') # dqn_pixel_atari('BreakoutNoFrameskip-v3') - async_pixel_atari('BreakoutNoFrameskip-v3') + # async_pixel_atari('BreakoutNoFrameskip-v3') # a3c_pixel_atari('BreakoutNoFrameskip-v3') diff --git a/network.py b/network.py index 659303b..d955e2a 100644 --- a/network.py +++ b/network.py @@ -18,9 +18,7 @@ class BasicNet: self.gpu = gpu and torch.cuda.is_available() self.LSTM = LSTM if self.gpu: - print 'Transferring network to GPU...' self.cuda() - print 'Network transferred.' def to_torch_variable(self, x, dtype='float32'): if not isinstance(x, torch.FloatTensor): @@ -74,9 +72,9 @@ class DuelingNet(BasicNet): # Starting of several network instances # Network for CartPole with value based methods -class FullyConnectedNet(nn.Module, VanillaNet): +class FCNet(nn.Module, VanillaNet): def __init__(self, dims, optimizer_fn=None, gpu=True): - super(FullyConnectedNet, self).__init__() + super(FCNet, self).__init__() self.fc1 = nn.Linear(dims[0], dims[1]) self.fc2 = nn.Linear(dims[1], dims[2]) self.fc3 = nn.Linear(dims[2], dims[3]) @@ -92,9 +90,9 @@ class FullyConnectedNet(nn.Module, VanillaNet): return y # Network for CartPole with dueling architecture -class DuelingFullyConnectedNet(nn.Module, DuelingNet): +class DuelingFCNet(nn.Module, DuelingNet): def __init__(self, dims, optimizer_fn=None, gpu=True): - super(DuelingFullyConnectedNet, self).__init__() + super(DuelingFCNet, self).__init__() self.fc1 = nn.Linear(dims[0], dims[1]) self.fc2 = nn.Linear(dims[1], dims[2]) self.fc_value = nn.Linear(dims[2], 1) @@ -109,10 +107,26 @@ class DuelingFullyConnectedNet(nn.Module, DuelingNet): phi = F.relu(self.fc2(y)) return phi + # Network for CartPole with actor critic +class ActorCriticFCNet(nn.Module, ActorCriticNet): + def __init__(self, + dims): + super(ActorCriticFCNet, self).__init__() + self.layer1 = nn.Linear(dims[0], dims[1]) + self.fc_actor = nn.Linear(dims[1], dims[2]) + self.fc_critic = nn.Linear(dims[1], 1) + BasicNet.__init__(self, None, False) + + def forward(self, x, update_LSTM=True): + x = self.to_torch_variable(x) + x = x.view(x.size(0), -1) + phi = self.layer1(x) + return phi + # Network for pixel Atari game with value based methods -class ConvNet(nn.Module, VanillaNet): +class NatureConvNet(nn.Module, VanillaNet): def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True): - super(ConvNet, self).__init__() + super(NatureConvNet, self).__init__() self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4) self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2) self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1) @@ -130,28 +144,10 @@ class ConvNet(nn.Module, VanillaNet): y = F.relu(self.fc4(y)) return self.fc5(y) -class NipsConvNet(nn.Module, VanillaNet): - def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True): - super(NipsConvNet, self).__init__() - self.conv1 = nn.Conv2d(in_channels, 16, kernel_size=8, stride=4) - self.conv2 = nn.Conv2d(16, 32, kernel_size=4, stride=2) - self.fc3 = nn.Linear(9 * 9 * 32, 256) - self.fc4 = nn.Linear(256, n_actions) - self.criterion = nn.MSELoss() - BasicNet.__init__(self, optimizer_fn, gpu) - - def forward(self, x): - x = self.to_torch_variable(x) - y = F.relu(self.conv1(x)) - y = F.relu(self.conv2(y)) - y = y.view(y.size(0), -1) - y = F.relu(self.fc3(y)) - return self.fc4(y) - # Network for pixel Atari game with dueling architecture -class DuelingConvNet(nn.Module, DuelingNet): +class DuelingNatureConvNet(nn.Module, DuelingNet): def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True): - super(DuelingConvNet, self).__init__() + super(DuelingNatureConvNet, self).__init__() self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4) self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2) self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1) @@ -170,45 +166,17 @@ class DuelingConvNet(nn.Module, DuelingNet): phi = F.relu(self.fc4(y)) return phi -# Network for CartPole with actor critic -class FCActorCriticNet(nn.Module, ActorCriticNet): - def __init__(self, - dims, - LSTM=False): - super(FCActorCriticNet, self).__init__() - if LSTM: - self.layer1 = nn.LSTMCell(dims[0], dims[1]) - else: - self.layer1 = nn.Linear(dims[0], dims[1]) - self.fc_actor = nn.Linear(dims[1], dims[2]) - self.fc_critic = nn.Linear(dims[1], 1) - BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=LSTM) - if LSTM: - self.h = self.to_torch_variable(np.zeros((1, dims[1]))) - self.c = self.to_torch_variable(np.zeros((1, dims[1]))) - def forward(self, x, update_LSTM=True): - x = self.to_torch_variable(x) - x = x.view(x.size(0), -1) - if self.LSTM: - h, c = self.layer1(x, (self.h, self.c)) - if update_LSTM: - self.h = h - self.c = c - phi = h - else: - phi = self.layer1(x) - return phi # Network for pixel Atari game with actor critic -class ConvActorCriticNet(nn.Module, ActorCriticNet): +class ActorCriticNatureConvNet(nn.Module, ActorCriticNet): def __init__(self, in_channels, n_actions, xentropy_weight=0.01, grad_threshold=40, gpu=True): - super(ConvActorCriticNet, self).__init__() + super(ActorCriticNatureConvNet, self).__init__() self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4) self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2) self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1) @@ -227,12 +195,12 @@ class ConvActorCriticNet(nn.Module, ActorCriticNet): y = y.view(y.size(0), -1) return F.elu(self.fc4(y)) -class OpenAIConvActorCriticNet(nn.Module, ActorCriticNet): +class OpenAIActorCriticConvNet(nn.Module, ActorCriticNet): def __init__(self, in_channels, n_actions, LSTM=False): - super(OpenAIConvActorCriticNet, self).__init__() + super(OpenAIActorCriticConvNet, self).__init__() self.conv1 = nn.Conv2d(in_channels, 32, 3, stride=2, padding=1) self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1) self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1) @@ -273,27 +241,18 @@ class OpenAIConvActorCriticNet(nn.Module, ActorCriticNet): class OpenAIConvNet(nn.Module, VanillaNet): def __init__(self, in_channels, - n_actions, - LSTM=False): + n_actions): super(OpenAIConvNet, self).__init__() self.conv1 = nn.Conv2d(in_channels, 32, 3, stride=2, padding=1) self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1) self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1) self.conv4 = nn.Conv2d(32, 32, 3, stride=2, padding=1) - self.LSTM = LSTM hidden_units = 256 - - if LSTM: - self.layer5 = nn.LSTMCell(32 * 3 * 3, hidden_units) - else: - self.layer5 = nn.Linear(32 * 3 * 3, hidden_units) - + self.layer5 = nn.Linear(32 * 3 * 3, hidden_units) self.fc6 = nn.Linear(hidden_units, n_actions) - BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=LSTM) - if LSTM: - self.h = self.to_torch_variable(np.zeros((1, hidden_units))) - self.c = self.to_torch_variable(np.zeros((1, hidden_units))) + + BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=False) def forward(self, x, update_LSTM=True): x = self.to_torch_variable(x) @@ -302,12 +261,5 @@ class OpenAIConvNet(nn.Module, VanillaNet): y = F.elu(self.conv3(y)) y = F.elu(self.conv4(y)) y = y.view(y.size(0), -1) - if self.LSTM: - h, c = self.layer5(y, (self.h, self.c)) - if update_LSTM: - self.h = h - self.c = c - phi = h - else: - phi = F.elu(self.layer5(y)) + phi = F.elu(self.layer5(y)) return self.fc6(phi)