diff --git a/agent/async_agent.py b/agent/async_agent.py index 5f5ac8e..66dc4c9 100644 --- a/agent/async_agent.py +++ b/agent/async_agent.py @@ -14,79 +14,79 @@ import pickle import os import time +def train(id, config, learning_network, target_network): + worker = config.worker(config, learning_network, target_network) + episode = 0 + rewards = [] + while not config.stop_signal.value: + steps, reward = worker.episode() + rewards.append(reward) + if len(rewards) > 100: rewards.pop(0) + config.logger.debug('worker %d, episode %d, return %f, avg return %f, episode steps %d, total steps %d' % ( + id, episode, rewards[-1], np.mean(rewards[-100:]), steps, config.total_steps.value)) + +def evaluate(config, task, learning_network): + test_rewards = [] + test_points = [] + worker = config.worker(config, learning_network, None) + while True: + steps = config.total_steps.value + if steps % config.test_interval == 0: + worker.worker_network.load_state_dict(learning_network.state_dict()) + with open('data/%s-%s-model-%s.bin' % ( + config.tag, config.worker.__name__, task.name), 'wb') as f: + pickle.dump(learning_network.state_dict(), f) + rewards = np.zeros(config.test_repetitions) + for i in range(config.test_repetitions): + rewards[i] = worker.episode(deterministic=True)[1] + config.logger.info('total steps: %d, averaged return per episode: %f(%f)' % \ + (steps, np.mean(rewards), np.std(rewards) / np.sqrt(config.test_repetitions))) + test_rewards.append(np.mean(rewards)) + test_points.append(steps) + with open('data/%s-%s-statistics-%s.bin' % ( + config.tag, config.worker.__name__, task.name), 'wb') as f: + pickle.dump([test_points, test_rewards], f) + if np.mean(rewards) > task.success_threshold: + config.stop_signal.value = True + break + class AsyncAgent: def __init__(self, config): self.config = config + self.config.steps_lock = mp.Lock() + self.config.network_lock = mp.Lock() + self.config.total_steps = mp.Value('i', 0) + self.config.stop_signal = mp.Value('i', False) + + def run(self): + config = self.config + task = config.task_fn() learning_network = config.network_fn() learning_network.share_memory() target_network = config.network_fn() target_network.share_memory() target_network.load_state_dict(learning_network.state_dict()) - self.task = config.task_fn() - - self.config.learning_network = learning_network - self.config.target_network = target_network - self.config.steps_lock = mp.Lock() - self.config.network_lock = mp.Lock() - self.config.total_steps = mp.Value('i', 0) - self.config.stop_signal = mp.Value('i', False) - - def train(self, id): - worker = self.config.worker(self.config) - episode = 0 - rewards = [] - while not self.config.stop_signal.value: - steps, reward = worker.episode() - rewards.append(reward) - if len(rewards) > 100: rewards.pop(0) - self.config.logger.debug('worker %d, episode %d, return %f, avg return %f, episode steps %d, total steps %d' % ( - id, episode, rewards[-1], np.mean(rewards[-100:]), steps, self.config.total_steps.value)) - - def save(self, file_name): - with open(file_name, 'wb') as f: - pickle.dump(self.config.learning_network.state_dict(), f) - - def evaluate(self, id): - test_rewards = [] - test_points = [] - worker = self.config.worker(self.config) - while True: - steps = self.config.total_steps.value - if steps % self.config.test_interval == 0: - worker.worker_network.load_state_dict(self.config.learning_network.state_dict()) - self.save('data/%s-%s-model-%s.bin' % ( - self.config.tag, self.config.worker.__name__, self.task.name)) - rewards = np.zeros(self.config.test_repetitions) - for i in range(self.config.test_repetitions): - rewards[i] = worker.episode(deterministic=True)[1] - self.config.logger.info('total steps: %d, averaged return per episode: %f(%f)' %\ - (steps, np.mean(rewards), np.std(rewards) / np.sqrt(self.config.test_repetitions))) - test_rewards.append(np.mean(rewards)) - test_points.append(steps) - with open('data/%s-%s-statistics-%s.bin' % ( - self.config.tag, self.config.worker.__name__, self.task.name - ), 'wb') as f: - pickle.dump([test_points, test_rewards], f) - if np.mean(rewards) > self.task.success_threshold: - self.config.stop_signal.value = True - break - - def run(self): os.environ['OMP_NUM_THREADS'] = '1' - procs = [mp.Process(target=self.train, args=(i, )) for i in range(self.config.num_workers)] - procs.append(mp.Process(target=self.evaluate, args=(self.config.num_workers, ))) + args = [(i, config, learning_network, target_network) for i in range(config.num_workers)] + args.append((config, task, learning_network)) + procs = [mp.Process(target=train, args=args[i]) for i in range(config.num_workers)] + procs.append(mp.Process(target=evaluate, args=args[-1])) for p in procs: p.start() while True: time.sleep(1) for i, p in enumerate(procs): - if not p.is_alive() and not self.config.stop_signal.value: - self.config.logger.warning('Worker %d exited unexpectedly.' % i) + if not p.is_alive() and not config.stop_signal.value: + config.logger.warning('Worker %d exited unexpectedly.' % i) p.terminate() - procs[i] = mp.Process(target=self.train, args=(i, )) + if i == config.num_workers: + target = evaluate + else: + target = train + procs[i] = mp.Process(target=target, args=args[i]) procs[i].start() self.config.logger.warning('Worker %d restarted.' % i) break - if self.config.stop_signal.value: + if config.stop_signal.value: break for p in procs: p.join() diff --git a/async_worker/actor_critic.py b/async_worker/actor_critic.py index 24267ae..93ffa6f 100644 --- a/async_worker/actor_critic.py +++ b/async_worker/actor_critic.py @@ -9,13 +9,14 @@ from torch.autograd import Variable import torch.nn as nn class AdvantageActorCritic: - def __init__(self, config): + def __init__(self, config, learning_network, target_network): self.config = config - self.optimizer = config.optimizer_fn(config.learning_network.parameters()) + self.optimizer = config.optimizer_fn(learning_network.parameters()) self.worker_network = config.network_fn() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(learning_network.state_dict()) self.task = config.task_fn() self.policy = config.policy_fn() + self.learning_network = learning_network def episode(self, deterministic=False): config = self.config @@ -51,12 +52,16 @@ class AdvantageActorCritic: GAE = torch.FloatTensor([[0]]) for i in reversed(range(len(pending))): prob, log_prob, value, action, reward = pending[i] - R = reward + config.discount * R - advantage = Variable(R) - value - GAE = config.discount * GAE + advantage.data - loss += 0.5 * advantage.pow(2) + if i == len(pending) - 1: + delta = reward + config.discount * R - value.data + else: + delta = reward + pending[i + 1][2].data - value.data + GAE = config.discount * config.gae_tau * GAE + delta loss += -log_prob.gather(1, Variable(torch.LongTensor([[action]]))) * Variable(GAE) - loss += 0.01 * torch.sum(torch.mul(prob, log_prob)) + loss += config.entropy_weight * torch.sum(torch.mul(prob, log_prob)) + + R = reward + config.discount * R + loss += 0.5 * (Variable(R) - value).pow(2) pending = [] self.worker_network.zero_grad() @@ -64,12 +69,12 @@ class AdvantageActorCritic: loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) for param, worker_param in zip( - config.learning_network.parameters(), self.worker_network.parameters()): + self.learning_network.parameters(), self.worker_network.parameters()): if param.grad is not None: break param._grad = worker_param.grad self.optimizer.step() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(self.learning_network.state_dict()) self.worker_network.reset(terminal) if terminal: diff --git a/async_worker/continuous_actor_critic.py b/async_worker/continuous_actor_critic.py index cd5bb17..c4b4ed1 100644 --- a/async_worker/continuous_actor_critic.py +++ b/async_worker/continuous_actor_critic.py @@ -9,13 +9,14 @@ from torch.autograd import Variable import torch.nn as nn class ContinuousAdvantageActorCritic: - def __init__(self, config): + def __init__(self, config, learning_network, target_network): self.config = config - self.optimizer = config.optimizer_fn(config.learning_network.parameters()) + self.optimizer = config.optimizer_fn(learning_network.parameters()) self.worker_network = config.network_fn() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(learning_network.state_dict()) self.task = config.task_fn() self.policy = config.policy_fn() + self.learning_network = learning_network def episode(self, deterministic=False): config = self.config @@ -34,6 +35,8 @@ class ContinuousAdvantageActorCritic: steps += 1 total_reward += reward + if not deterministic: + reward = np.clip(reward, -1, 1) if deterministic: if terminal: @@ -54,29 +57,36 @@ class ContinuousAdvantageActorCritic: GAE = torch.FloatTensor([[0]]) for i in reversed(range(len(pending))): mean, var, value, action, reward = pending[i] - R = reward + config.discount * R - advantage = Variable(R) - value - GAE = config.discount * config.gae_tau * GAE + advantage.data - loss += 0.5 * advantage.pow(2) + if i == len(pending) - 1: + delta = reward + config.discount * R - value.data + else: + delta = reward + pending[i + 1][2].data - value.data + GAE = config.discount * config.gae_tau * GAE + delta + action = Variable(torch.FloatTensor([action])) prob_part1 = (-(action - mean).pow(2) / (2 * var)).exp() prob_part2 = 1 / (2 * var * pi.expand_as(var)).sqrt() prob = prob_part1 * prob_part2 log_prob = prob.log() loss += -torch.sum(log_prob) * Variable(GAE) - entropy = 0.5 * (1.0 + (var + 2 * pi.expand_as(var)).log()).sum() + entropy = 0.5 * (1.0 + (var * 2 * pi.expand_as(var)).log()).sum() loss += config.entropy_weight * entropy + R = reward + config.discount * R + loss += 0.5 * (Variable(R) - value).pow(2) + pending = [] self.worker_network.zero_grad() + self.optimizer.zero_grad() loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) - self.optimizer.zero_grad() for param, worker_param in zip( - config.learning_network.parameters(), self.worker_network.parameters()): - param._grad = worker_param.grad.clone() + self.learning_network.parameters(), self.worker_network.parameters()): + if param.grad is not None: + break + param._grad = worker_param.grad self.optimizer.step() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(self.learning_network.state_dict()) self.worker_network.reset(terminal) if terminal: diff --git a/async_worker/n_step_q.py b/async_worker/n_step_q.py index 8fc9875..c0d30a8 100644 --- a/async_worker/n_step_q.py +++ b/async_worker/n_step_q.py @@ -9,13 +9,15 @@ from torch.autograd import Variable import torch.nn as nn class NStepQLearning: - def __init__(self, config): + def __init__(self, config, learning_network, target_network): self.config = config - self.optimizer = config.optimizer_fn(config.learning_network.parameters()) + self.optimizer = config.optimizer_fn(learning_network.parameters()) self.worker_network = config.network_fn() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(learning_network.state_dict()) self.task = config.task_fn() self.policy = config.policy_fn() + self.learning_network = learning_network + self.target_network = target_network def episode(self, deterministic=False): config = self.config @@ -47,7 +49,7 @@ class NStepQLearning: if terminal: R = torch.FloatTensor([[0]]) else: - R, _ = config.target_network.predict( + R, _ = self.target_network.predict( np.stack([next_state])).data.max(1) for i in reversed(range(len(pending))): @@ -61,12 +63,12 @@ class NStepQLearning: loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) for param, worker_param in zip( - config.learning_network.parameters(), self.worker_network.parameters()): + self.learning_network.parameters(), self.worker_network.parameters()): if param.grad is not None: break param._grad = worker_param.grad self.optimizer.step() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(self.learning_network.state_dict()) self.worker_network.reset(terminal) if terminal: @@ -74,6 +76,6 @@ class NStepQLearning: state = next_state if config.total_steps.value % config.target_network_update_freq == 0: - config.target_network.load_state_dict(config.learning_network.state_dict()) + self.target_network.load_state_dict(self.learning_network.state_dict()) return steps, total_reward \ No newline at end of file diff --git a/async_worker/one_step_q.py b/async_worker/one_step_q.py index 4e60a17..fd41862 100644 --- a/async_worker/one_step_q.py +++ b/async_worker/one_step_q.py @@ -9,13 +9,15 @@ from torch.autograd import Variable import torch.nn as nn class OneStepQLearning: - def __init__(self, config): + def __init__(self, config, learning_network, target_network): self.config = config - self.optimizer = config.optimizer_fn(config.learning_network.parameters()) + self.optimizer = config.optimizer_fn(learning_network.parameters()) self.worker_network = config.network_fn() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(learning_network.state_dict()) self.task = config.task_fn() self.policy = config.policy_fn() + self.learning_network = learning_network + self.target_network = target_network def episode(self, deterministic=False): config = self.config @@ -46,7 +48,7 @@ class OneStepQLearning: loss = 0 for i in range(len(pending)): q, action, reward, next_state = pending[i] - q_next, _ = config.target_network.predict(np.stack([next_state])).data.max(1) + q_next, _ = self.target_network.predict(np.stack([next_state])).data.max(1) if terminal and i == len(pending) - 1: q_next = torch.FloatTensor([[0]]) q_next = config.discount * q_next + reward @@ -59,12 +61,12 @@ class OneStepQLearning: loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) for param, worker_param in zip( - config.learning_network.parameters(), self.worker_network.parameters()): + self.learning_network.parameters(), self.worker_network.parameters()): if param.grad is not None: break param._grad = worker_param.grad self.optimizer.step() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(self.learning_network.state_dict()) self.worker_network.reset(terminal) if terminal: @@ -72,6 +74,6 @@ class OneStepQLearning: state = next_state if config.total_steps.value % config.target_network_update_freq == 0: - config.target_network.load_state_dict(config.learning_network.state_dict()) + self.target_network.load_state_dict(self.learning_network.state_dict()) return steps, total_reward \ No newline at end of file diff --git a/async_worker/one_step_sarsa.py b/async_worker/one_step_sarsa.py index 942ddc2..f6d6464 100644 --- a/async_worker/one_step_sarsa.py +++ b/async_worker/one_step_sarsa.py @@ -9,13 +9,15 @@ from torch.autograd import Variable import torch.nn as nn class OneStepSarsa: - def __init__(self, config): + def __init__(self, config, learning_network, target_network): self.config = config - self.optimizer = config.optimizer_fn(config.learning_network.parameters()) + self.optimizer = config.optimizer_fn(learning_network.parameters()) self.worker_network = config.network_fn() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(learning_network.state_dict()) self.task = config.task_fn() self.policy = config.policy_fn() + self.learning_network = learning_network + self.target_network = target_network def episode(self, deterministic=False): config = self.config @@ -49,7 +51,7 @@ class OneStepSarsa: loss = 0 for i in range(len(pending)): q, action, reward, next_state, next_action = pending[i] - q_next = config.target_network.predict(np.stack([next_state])).data + q_next = self.target_network.predict(np.stack([next_state])).data if terminal and i == len(pending) - 1: q_next = torch.FloatTensor([[0]]) else: @@ -64,12 +66,12 @@ class OneStepSarsa: loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) for param, worker_param in zip( - config.learning_network.parameters(), self.worker_network.parameters()): + self.learning_network.parameters(), self.worker_network.parameters()): if param.grad is not None: break param._grad = worker_param.grad self.optimizer.step() - self.worker_network.load_state_dict(config.learning_network.state_dict()) + self.worker_network.load_state_dict(self.learning_network.state_dict()) self.worker_network.reset(terminal) if terminal: @@ -79,6 +81,6 @@ class OneStepSarsa: action = next_action if config.total_steps.value % config.target_network_update_freq == 0: - config.target_network.load_state_dict(config.learning_network.state_dict()) + self.target_network.load_state_dict(self.learning_network.state_dict()) return steps, total_reward diff --git a/main.py b/main.py index 1864c21..6701083 100644 --- a/main.py +++ b/main.py @@ -31,8 +31,8 @@ def async_cart_pole(): config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) config.network_fn = lambda: FCNet([4, 50, 200, 2]) config.policy_fn = lambda: GreedyPolicy(epsilon=0.5, final_step=5000, min_epsilon=0.1) - # config.worker = OneStepQLearning - config.worker = NStepQLearning + config.worker = OneStepQLearning + # config.worker = NStepQLearning # config.worker = OneStepSarsa config.discount = 0.99 config.target_network_update_freq = 200 @@ -56,7 +56,7 @@ def a3c_cart_pole(): config.max_episode_length = 200 config.num_workers = 16 config.update_interval = 6 - config.test_interval = 100 + config.test_interval = 1 config.test_repetitions = 30 config.logger = Logger('./log', gym.logger) config.gae_tau = 1.0 @@ -68,7 +68,7 @@ def a3c_pendulum(): config = Config() config.task_fn = lambda: Pendulum() task = config.task_fn() - config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) + config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001) config.network_fn = lambda: ContinuousActorCriticNet( task.env.observation_space.shape[0], 64, task.env.action_space.shape[0]) config.policy_fn = lambda: GaussianPolicy() @@ -78,7 +78,7 @@ def a3c_pendulum(): config.num_workers = 16 config.update_interval = 20 config.test_interval = 1 - config.test_repetitions = 50 + config.test_repetitions = 1 config.entropy_weight = 0.0001 config.logger = Logger('./log', gym.logger) agent = AsyncAgent(config) @@ -120,12 +120,12 @@ def async_pixel_atari(name): epsilons=[0.7, 0.7, 0.7], final_step=2000000, min_epsilons=[0.1, 0.01, 0.5], probs=[0.4, 0.3, 0.3]) # config.worker = OneStepSarsa - config.worker = NStepQLearning - # config.worker = OneStepQLearning + # config.worker = NStepQLearning + config.worker = OneStepQLearning config.discount = 0.99 config.target_network_update_freq = 10000 config.max_episode_length = 10000 - config.num_workers = 16 + config.num_workers = 10 config.update_interval = 20 config.test_interval = 50000 config.test_repetitions = 1 @@ -145,7 +145,7 @@ def a3c_pixel_atari(name): config.worker = AdvantageActorCritic config.discount = 0.99 config.max_episode_length = 10000 - config.num_workers = 16 + config.num_workers = 10 config.update_interval = 20 config.test_interval = 50000 config.test_repetitions = 1 @@ -208,12 +208,12 @@ if __name__ == '__main__': # dqn_cart_pole() # async_cart_pole() - a3c_cart_pole() + # a3c_cart_pole() # a3c_pendulum() # dqn_pixel_atari('PongNoFrameskip-v3') # async_pixel_atari('PongNoFrameskip-v3') - # a3c_pixel_atari('PongNoFrameskip-v3') + a3c_pixel_atari('PongNoFrameskip-v3') # dqn_pixel_atari('BreakoutNoFrameskip-v3') # async_pixel_atari('BreakoutNoFrameskip-v3')