From 4786bb8990fbaa5ebeca148aa352e290e27c0057 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Thu, 11 May 2017 21:30:04 -0600 Subject: [PATCH] Update Readme --- README.md | 9 ++++++++- async_agent.py | 28 +++++++++++++++++++--------- dqn_agent.py | 3 ++- main.py | 33 +++++++++++++++++++++++++++++++++ task.py | 19 +++++++------------ 5 files changed, 69 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 3ef3eda..63fa2fd 100644 --- a/README.md +++ b/README.md @@ -1 +1,8 @@ -# DeepRL by PyTorch +#DeepRL +> Highly modularized implementation of popular deep RL algorithms powered by PyTorch +* Deep Q-Learning +* Asynchronous One-Step Q-Learning + +>Benchmarked by classical control tasks (CartPole, LunarLander). Atari games will make it difficult to replicate in a regular laptop without a good GPU. However it's fairly easy to adapt the components to fit Atari games. + +>Try it out from ```main.py```! \ No newline at end of file diff --git a/async_agent.py b/async_agent.py index ff526f4..d607f35 100644 --- a/async_agent.py +++ b/async_agent.py @@ -13,7 +13,7 @@ from network import * class AsyncAgent: def __init__(self, task_fn, network_fn, optimizer_fn, policy_fn, discount, step_limit, - target_network_update_freq, n_workers, batch_size, test_interval): + target_network_update_freq, n_workers, batch_size, test_interval, test_repeats): self.network_fn = network_fn self.learning_network = network_fn() self.learning_network.share_memory() @@ -35,14 +35,14 @@ class AsyncAgent: self.n_workers = n_workers self.batch_size = batch_size self.test_interval = test_interval + self.test_repeats = test_repeats - def deterministic_episode(self, task): + def deterministic_episode(self, task, network): state = np.asarray([task.reset()]) total_rewards = 0 steps = 0 while True and steps < self.step_limit: - with self.network_lock: - action_values = self.learning_network.predict(state) + action_values = network.predict(state) steps += 1 action = np.argmax(action_values.flatten()) state, reward, terminal, _ = task.step(action) @@ -68,10 +68,14 @@ class AsyncAgent: terminal = True episode = 0 episode_steps = 0 + episode_return = 0 while True and not self.stop_signal.value: batch_states, batch_actions, batch_rewards = [], [], [] if terminal: + if id == 0: + print 'worker %d, episode %d, return %f' % (id, episode, episode_return) episode_steps = 0 + episode_return = 0 episode += 1 policy.update_epsilon() terminal = False @@ -86,6 +90,7 @@ class AsyncAgent: action = policy.sample(value.flatten()) batch_actions.append(action) state, reward, terminal, _ = task.step(action) + episode_return += reward state = state.reshape([1, -1]) if not terminal: with self.network_lock: @@ -93,6 +98,9 @@ class AsyncAgent: reward += self.discount * q_next batch_rewards.append(reward) + if episode_steps > self.step_limit: + terminal = True + worker_network.zero_grad() worker_network.gradient(np.vstack(batch_states), batch_actions, batch_rewards) self.async_update(worker_network, optimizer) @@ -106,13 +114,15 @@ class AsyncAgent: procs = [mp.Process(target=self.worker, args=(i, )) for i in range(self.n_workers)] for p in procs: p.start() task = self.task_fn() + test_network = self.network_fn() while True: if self.total_steps.value % self.test_interval == 0: - test_repeats = 5 - rewards = np.zeros(test_repeats) - for i in range(test_repeats): - rewards[i] = self.deterministic_episode(task) - print 'total stpes: %d, test process epsidoe reward: %f' %\ + with self.network_lock: + test_network.load_state_dict(self.learning_network.state_dict()) + rewards = np.zeros(self.test_repeats) + for i in range(self.test_repeats): + rewards[i] = self.deterministic_episode(task, test_network) + print 'total steps: %d, averaged return per episode: %f' %\ (self.total_steps.value, np.mean(rewards)) if np.mean(rewards) > task.success_threshold: self.stop_signal.value = True diff --git a/dqn_agent.py b/dqn_agent.py index e7990d8..5b3829b 100644 --- a/dqn_agent.py +++ b/dqn_agent.py @@ -62,7 +62,8 @@ class DQNAgent: rewards.append(reward) if len(rewards) > window_size: reward = np.mean(rewards[-window_size:]) - print 'episode %d: %f' % (ep, reward) + print 'episode %d, epsilon %f, reward %f' % ( + ep, self.policy.epsilon, reward) if reward > self.task.success_threshold: break diff --git a/main.py b/main.py index a9fdd03..0bea53d 100644 --- a/main.py +++ b/main.py @@ -13,6 +13,37 @@ def async_cart_pole(): config['n_workers'] = 8 config['batch_size'] = 5 config['test_interval'] = 500 + config['test_repeats'] = 5 + agent = AsyncAgent(**config) + agent.run() + +# Mountain Car is fairly unstable +def dqn_mountain_car(): + config = dict() + config['task_fn'] = lambda: MountainCar() + config['optimizer_fn'] = lambda params: torch.optim.SGD(params, 0.001) + config['network_fn'] = lambda optimizer_fn: FullyConnectedNet([2, 50, 200, 3], optimizer_fn) + config['policy_fn'] = lambda: GreedyPolicy(epsilon=0.5, end_episode=500, min_epsilon=0.1) + config['replay_fn'] = lambda: Replay(memory_size=10000, batch_size=10) + config['discount'] = 0.99 + config['target_network_update_freq'] = 1000 + config['step_limit'] = 5000 + agent = DQNAgent(**config) + agent.run() + +def async_lunar_lander(): + config = dict() + config['task_fn'] = lambda: LunarLander() + config['optimizer_fn'] = lambda params: torch.optim.Adam(params, 0.001) + config['network_fn'] = lambda: FullyConnectedNet([8, 50, 200, 4]) + config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, end_episode=2000, min_epsilon=0.05) + config['discount'] = 0.99 + config['target_network_update_freq'] = 200 + config['step_limit'] = 5000 + config['n_workers'] = 8 + config['batch_size'] = 10 + config['test_interval'] = 1000 + config['test_repeats'] = 5 agent = AsyncAgent(**config) agent.run() @@ -31,4 +62,6 @@ def dqn_cart_pole(): if __name__ == '__main__': async_cart_pole() + # async_lunar_lander() # dqn_cart_pole() + # dqn_mountain_car() diff --git a/task.py b/task.py index 60e1503..25efd48 100644 --- a/task.py +++ b/task.py @@ -3,11 +3,8 @@ # Permission given to modify the code as long as you keep this # # declaration at the top # ####################################################################### - import gym import sys -from dqn_agent import * -import torch.optim class BasicTask: def transfer_state(self, state): @@ -22,21 +19,12 @@ class BasicTask: return next_state, reward, done, info class MountainCar(BasicTask): - state_space_size = 2 - action_space_size = 3 name = 'MountainCar-v0' success_threshold = -110 - discount = 0.99 - step_limit = 5000 - target_network_update_freq = 1000 def __init__(self): self.env = gym.make(self.name) self.env._max_episode_steps = sys.maxsize - self.optimizer_fn = lambda params: torch.optim.SGD(params, 0.001) - self.network_fn = lambda optimizer_fn: FullyConnectedNet([self.state_space_size, 50, 200, self.action_space_size], optimizer_fn) - self.policy_fn = lambda: GreedyPolicy(epsilon=0.5, decay_factor=0.95, min_epsilon=0.1) - self.replay_fn = lambda: Replay(memory_size=10000, batch_size=10) class CartPole(BasicTask): name = 'CartPole-v0' @@ -44,3 +32,10 @@ class CartPole(BasicTask): def __init__(self): self.env = gym.make(self.name) + +class LunarLander(BasicTask): + name = 'LunarLander-v2' + success_threshold = 200 + + def __init__(self): + self.env = gym.make(self.name) \ No newline at end of file