diff --git a/async_agent.py b/async_agent.py index b43f532..4b83c7d 100644 --- a/async_agent.py +++ b/async_agent.py @@ -25,7 +25,8 @@ class AsyncAgent: n_workers, batch_size, test_interval, - test_repeats, + test_repetitions, + history_length, logger): self.network_fn = network_fn self.learning_network = network_fn() @@ -49,23 +50,27 @@ class AsyncAgent: self.n_workers = n_workers self.batch_size = batch_size self.test_interval = test_interval - self.test_repeats = test_repeats + self.test_repetitions = test_repetitions self.logger = logger + self.history_length = history_length def deterministic_episode(self, task, network): state = np.asarray([task.reset()]) total_rewards = 0 steps = 0 terminal = False + buffer = [state] * self.history_length while not terminal and steps < self.step_limit: - action_values = network.predict(state) + state = task.normalize_state(np.vstack(buffer)) + action_values = network.predict(np.reshape(state, (1, ) + state.shape)) steps += 1 action = np.argmax(action_values.flatten()) state, reward, terminal, _ = task.step(action) + buffer.pop(0) + buffer.append(state) total_rewards += reward if terminal: break - state = state.reshape([1, -1]) return total_rewards def async_update(self, worker_network, optimizer): @@ -85,29 +90,37 @@ class AsyncAgent: episode = 0 episode_steps = 0 episode_return = 0 + episode_returns = [0] while True and not self.stop_signal.value: batch_states, batch_actions, batch_rewards = [], [], [] if terminal: if id == 0: - self.logger.debug('worker %d, episode %d, return %f' % (id, episode, episode_return)) + self.logger.info('episode %d, epsilon %f, return %f, avg return %f, total steps %d' % ( + episode, policy.epsilon, episode_return, np.mean(episode_returns[-100: ]), + self.total_steps.value)) episode_steps = 0 + episode_returns.append(episode_return) episode_return = 0 episode += 1 terminal = False state = task.reset() - state = state.reshape([1, -1]) - value = worker_network.predict(state) + buffer = [state] * self.history_length + state = task.normalize_state(np.vstack(buffer)) + value = worker_network.predict(np.reshape(state, (1, ) + state.shape)) action = policy.sample(value.flatten()) while not terminal and len(batch_states) < self.batch_size: episode_steps += 1 - self.total_steps.value += 1 + with self.steps_lock: + self.total_steps.value += 1 batch_states.append(state) batch_actions.append(action) state, reward, terminal, _ = task.step(action) batch_rewards.append(reward) episode_return += reward - state = state.reshape([1, -1]) - value = worker_network.predict(state) + buffer.pop(0) + buffer.append(state) + state = task.normalize_state(np.vstack(buffer)) + value = worker_network.predict(np.reshape(state, (1, ) + state.shape)) action = policy.sample(value.flatten()) policy.update_epsilon() @@ -118,7 +131,7 @@ class AsyncAgent: terminal = True worker_network.zero_grad() - worker_network.gradient(np.vstack(batch_states), batch_actions, batch_rewards) + worker_network.gradient(np.asarray(batch_states), batch_actions, batch_rewards) self.async_update(worker_network, optimizer) worker_network.load_state_dict(self.learning_network.state_dict()) @@ -136,8 +149,8 @@ class AsyncAgent: if steps % self.test_interval == 0: with self.network_lock: test_network.load_state_dict(self.learning_network.state_dict()) - rewards = np.zeros(self.test_repeats) - for i in range(self.test_repeats): + rewards = np.zeros(self.test_repetitions) + for i in range(self.test_repetitions): rewards[i] = self.deterministic_episode(task, test_network) self.logger.info('total steps: %d, averaged return per episode: %f' %\ (steps, np.mean(rewards))) diff --git a/bootstrap.py b/bootstrap.py index e60c241..910a788 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -11,7 +11,8 @@ def NStepQLearning(batch_states, batch_actions, batch_rewards, reward = 0 else: with agent.network_lock: - reward = np.max(agent.target_network.predict(tailing_state)) + reward = np.max(agent.target_network.predict( + np.reshape(tailing_state, (1, ) + tailing_state.shape))) rewards = [] for r in reversed(batch_rewards): reward = r + agent.discount * reward @@ -22,7 +23,7 @@ def OneStepQLearning(batch_states, batch_actions, batch_rewards, tailing_state, tailing_action, terminal, agent): batch_states.append(tailing_state) with agent.network_lock: - q_next = agent.target_network.predict(np.vstack(batch_states[1:])) + q_next = agent.target_network.predict(np.asarray(batch_states[1:])) q_next = np.max(q_next, axis=1) if terminal: q_next[-1] = 0 @@ -35,7 +36,7 @@ def OneStepSarsa(batch_states, batch_actions, batch_rewards, batch_states.append(tailing_state) batch_actions.append(tailing_action) with agent.network_lock: - q_next = agent.target_network.predict(np.vstack(batch_states[1:])) + q_next = agent.target_network.predict(np.asarray(batch_states[1:])) q_next = q_next[np.arange(len(batch_actions[1:])), batch_actions[1:]] if terminal: q_next[-1] = 0 diff --git a/dqn_agent.py b/dqn_agent.py index 7f77442..86ff93a 100644 --- a/dqn_agent.py +++ b/dqn_agent.py @@ -44,33 +44,35 @@ class DQNAgent: self.process = psutil.Process(os.getpid()) self.test_interval = test_interval self.test_repetitions = test_repetitions - - def get_state(self, history_buffer): - return np.vstack(history_buffer) + self.history_buffer = None def episode(self, deterministic=False): episode_start_time = time.time() state = self.task.reset() - history_buffer = [state] * self.history_length + if self.history_buffer is None: + self.history_buffer = [np.zeros_like(state)] * self.history_length + else: + self.history_buffer.pop(0) + self.history_buffer.append(state) + state = np.vstack(self.history_buffer) total_reward = 0.0 steps = 0 while not self.step_limit or steps < self.step_limit: - state = self.get_state(history_buffer) - state = self.task.normalize_state(state) - value = self.learning_network.predict(np.reshape(state, (1, ) + state.shape)) + value = self.learning_network.predict(np.stack([self.task.normalize_state(state)])) if deterministic: action = np.argmax(value.flatten()) else: action = self.policy.sample(value.flatten()) next_state, reward, done, info = self.task.step(action) - history_buffer.pop(0) - history_buffer.append(next_state) + self.history_buffer.pop(0) + self.history_buffer.append(next_state) + next_state = np.vstack(self.history_buffer) if not deterministic: - next_state = self.get_state(history_buffer) self.replay.feed([state, action, reward, next_state, int(done)]) self.total_steps += 1 total_reward += reward steps += 1 + state = next_state if done: break if not deterministic and self.total_steps > self.explore_steps: @@ -111,7 +113,7 @@ class DQNAgent: if ep % self.test_interval == 0: self.logger.info('Testing...') - self.save('data/dqn-episode-%d.bin' % (ep)) + self.save('data/dqn-model.bin') test_rewards = [] for _ in range(self.test_repetitions): test_rewards.append(self.episode(True)) diff --git a/main.py b/main.py index 4efa602..ab3f427 100644 --- a/main.py +++ b/main.py @@ -6,18 +6,20 @@ def async_cart_pole(): config = dict() config['task_fn'] = lambda: CartPole() config['optimizer_fn'] = lambda params: torch.optim.SGD(params, 0.001) - config['network_fn'] = lambda: FullyConnectedNet([4, 50, 200, 2]) + config['network_fn'] = lambda: FullyConnectedNet([8, 50, 200, 2]) config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=5000, min_epsilon=0.1) - # config['bootstrap_fn'] = OneStepQLearning + config['bootstrap_fn'] = OneStepQLearning # config['bootstrap_fn'] = NStepQLearning - config['bootstrap_fn'] = OneStepSarsa + # config['bootstrap_fn'] = OneStepSarsa config['discount'] = 0.99 config['target_network_update_freq'] = 200 config['step_limit'] = 300 config['n_workers'] = 8 config['batch_size'] = 5 - config['test_interval'] = 500 - config['test_repeats'] = 5 + config['test_interval'] = 4000 + config['test_repetitions'] = 50 + config['history_length'] = 2 + config['logger'] = gym.logger agent = AsyncAgent(**config) agent.run() @@ -34,7 +36,7 @@ def async_lunar_lander(): config['n_workers'] = 8 config['batch_size'] = 10 config['test_interval'] = 1000 - config['test_repeats'] = 5 + config['test_repetitions'] = 5 agent = AsyncAgent(**config) agent.run() @@ -69,7 +71,7 @@ def actor_critic_cart_pole(): config['n_workers'] = 10 config['batch_size'] = 5 config['test_interval'] = 50000 - config['test_repeats'] = 5 + config['test_repetitions'] = 5 config['logger'] = gym.logger agent = AsyncAgent(**config) agent.run() @@ -94,6 +96,29 @@ def dqn_pixel_atari(name): agent = DQNAgent(**config) agent.run() +def async_pixel_atari(name): + config = dict() + history_length = 4 + n_actions = 6 + config['task_fn'] = lambda: PixelAtari(name, 30, 4) + config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) + config['network_fn'] = lambda : ConvNet(history_length, n_actions, gpu=True) + config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) + config['bootstrap_fn'] = OneStepQLearning + # config['bootstrap_fn'] = NStepQLearning + # config['bootstrap_fn'] = OneStepSarsa + config['discount'] = 0.99 + config['target_network_update_freq'] = 10000 + config['step_limit'] = 0 + config['n_workers'] = 1 + config['batch_size'] = 32 + config['test_interval'] = 50000 + config['test_repetitions'] = 50 + config['history_length'] = history_length + config['logger'] = gym.logger + agent = AsyncAgent(**config) + agent.run() + if __name__ == '__main__': # gym.logger.setLevel(logging.DEBUG) gym.logger.setLevel(logging.INFO) @@ -104,3 +129,4 @@ if __name__ == '__main__': # actor_critic_cart_pole() # dqn_cart_pole() dqn_pixel_atari('BreakoutNoFrameskip-v3') + # async_pixel_atari('BreakoutNoFrameskip-v3') diff --git a/network.py b/network.py index 8965426..6a8ed72 100644 --- a/network.py +++ b/network.py @@ -160,7 +160,7 @@ class ConvNet(nn.Module): self.optimizer.step() def gradient(self, x, actions, targets): - y = self.forward(x) + y = self.forward(self.to_torch_variable(x)) actions = self.to_torch_variable(actions, 'int64').unsqueeze(1) targets = self.to_torch_variable(targets).unsqueeze(1) y = y.gather(1, actions)