diff --git a/README.md b/README.md index 40932d1..8f85185 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Implemented algorithms: * Async One-Step Sarsa * Async N-Step Q-Learning * Continuous A3C -* Deep Deterministic Policy Gradient (DDPG) +* Distributed Deep Deterministic Policy Gradient (Distributed DDPG, aka D3PG) * Hybrid Reward Architecture (HRA) -* Distributed Proximal Policy Optimization (DPPO) +* Parallelized Proximal Policy Optimization (P3O, similar to DPPO) # Curves > Curves for CartPole are trivial so I didn't place it here. There isn't any fixed random seed. @@ -28,6 +28,8 @@ Xeon E5-2620 v3 and Titan X. For Breakout, test is triggered every 1000 episodes In total, 16M frames cost about 4 days and 10 hours. For Pong, test is triggered every 10 episodes with no repetition. In total, 4M frames cost about 18 hours. +I referred this [repo](https://github.com/transedward/pytorch-dqn). + ## Discrete A3C ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/A3C-Pong.png) @@ -40,6 +42,8 @@ Training of A3C took about 2 hours (16 processes) in a server with two Xeon E5-2 Those value based async methods do work but I don't know how to make them stable. This is the test curve. Test is triggered in a separate deterministic test process every 50K frames. +I referred this [repo](https://github.com/ikostrikov/pytorch-a3c) for the parallelization. + ## Continuous A3C ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/Continuous-A3C.png) @@ -47,23 +51,31 @@ For continuous A3C and DPPO, I use fixed unit variance rather than a separate he Of course you can also use another head to output variance. In that case, a good practice is to bound your mean while leave variance unbounded, which is also included in the implementation. -## DDPG +## D3PG ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DDPG.png) -Extra caution is necessary when computing gradients, the [repo](https://github.com/ghliu/pytorch-ddpg) I referred -seems to have critical bugs. DDPG is not very stable. +Extra caution is necessary when computing gradients. The [repo](https://github.com/ghliu/pytorch-ddpg) I referred +for DDPG is wrong in computing the deterministic gradients at least at this [commit](https://github.com/ghliu/pytorch-ddpg/tree/ffea335ee53f2ff90b6d7eaf9d0cee705270c0f1). +Theoretically I believe that implementation should work, but in practice it doesn't work. Even this is PyTorch you need to manually deal with gradients in this case. +DDPG is not very stable. -## DPPO +Setting the number of workers to 1 will reduce the implementation to exact DDPG. I have to adopt the most straightforward distribution method, as +P3O and A3C style distribution doesn't work for DDPG. The figures were done with 6 workers. -![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DPPO.png) + +## P3O + +![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/P3O.png) The difference between my implementation and [DeepMind's DPPO](https://arxiv.org/abs/1707.02286) is: 1. PPO stands for different algorithms. 2. I use a much simpler A3C-like synchronization protocol. -The body of PPO is based on [this](https://github.com/alexis-jacq/Pytorch-DPPO), however that implementation has some - critical bugs. +The body of PPO is based on this [repo](https://github.com/alexis-jacq/Pytorch-DPPO). +However that implementation has two critical bugs at least at this [commit](https://github.com/ghliu/pytorch-ddpg/tree/ffea335ee53f2ff90b6d7eaf9d0cee705270c0f1). +Its computation of the clipped loss is correct with one-dimensional action by accident, +but is wrong with high-dimensional action. And its computation of entropy is wrong in any case. I use 8 threads and a two tanh hidden layer network, each hidden layer has 64 hidden units. @@ -81,7 +93,7 @@ Detailed usage and all training parameters can be found in ```main.py```. And you need to create following directories before running the program: ``` cd DeepRL -mkdir data log evaluation_log +mkdir data log ``` # References @@ -98,7 +110,3 @@ mkdir data log evaluation_log * [Trust Region Policy Optimization](https://arxiv.org/abs/1502.05477) * [Proximal Policy Optimization Algorithms](https://arxiv.org/abs/1707.06347) * [Emergence of Locomotion Behaviours in Rich Environments](https://arxiv.org/abs/1707.02286) -* [transedward/pytorch-dqn](https://github.com/transedward/pytorch-dqn) -* [ikostrikov/pytorch-a3c](https://github.com/ikostrikov/pytorch-a3c) -* [ghliu/pytorch-ddpg](https://github.com/ghliu/pytorch-ddpg) -* [alexis-jacq/Pytorch-DPPO](https://github.com/alexis-jacq/Pytorch-DPPO) diff --git a/agent/__init__.py b/agent/__init__.py index b77c3eb..8462c94 100644 --- a/agent/__init__.py +++ b/agent/__init__.py @@ -1,3 +1,2 @@ from .async_agent import * -from .DDPG_agent import * from .DQN_agent import * diff --git a/agent/async_agent.py b/agent/async_agent.py index de1eba8..b5e50d7 100644 --- a/agent/async_agent.py +++ b/agent/async_agent.py @@ -32,7 +32,6 @@ def evaluate(config, task, learning_network, extra): test_wall_times = [] initial_time = time.time() worker = config.worker(config, learning_network, extra) - # config.logger = Logger('./evaluation_log', gym.logger) while True: steps = config.total_steps.value if config.test_interval and steps % config.test_interval == 0: @@ -51,7 +50,7 @@ def evaluate(config, task, learning_network, extra): with open('data/%s-%s-statistics-%s.bin' % ( config.tag, config.worker.__name__, task.name), 'wb') as f: pickle.dump([test_rewards, test_points, test_wall_times], f) - if np.mean(rewards) > task.success_threshold or (config.max_steps and steps >= config.max_steps): + if np.mean(rewards) >= config.success_threshold or (config.max_steps and steps >= config.max_steps): config.stop_signal.value = True break @@ -75,10 +74,14 @@ class AsyncAgent: target_network.share_memory() target_network.load_state_dict(learning_network.state_dict()) extra = target_network - elif config.worker == ContinuousAdvantageActorCritic or config.worker == ProximalPolicyOptimization: + elif config.worker == ContinuousAdvantageActorCritic \ + or config.worker == ProximalPolicyOptimization\ + or config.worker == DeterministicPolicyGradient: state_normalizer = StaticNormalizer(task.state_dim) reward_normalizer = StaticNormalizer(1) extra = [state_normalizer, reward_normalizer] + if config.worker == DeterministicPolicyGradient: + extra.append(config.replay_fn()) else: extra = None args = [(i, config, learning_network, extra) for i in range(config.num_workers)] diff --git a/async_worker/__init__.py b/async_worker/__init__.py index ee5e06e..9a2160d 100644 --- a/async_worker/__init__.py +++ b/async_worker/__init__.py @@ -3,4 +3,5 @@ from .continuous_actor_critic import * from .n_step_q import * from .one_step_sarsa import * from .one_step_q import * -from .ppo import * \ No newline at end of file +from .ppo import * +from .dpg import * \ No newline at end of file diff --git a/async_worker/actor_critic.py b/async_worker/actor_critic.py index 423452c..9ad9417 100644 --- a/async_worker/actor_critic.py +++ b/async_worker/actor_critic.py @@ -7,6 +7,7 @@ import numpy as np import torch from torch.autograd import Variable import torch.nn as nn +from utils import * class AdvantageActorCritic: def __init__(self, config, learning_network, target_network): @@ -55,7 +56,7 @@ class AdvantageActorCritic: if i == len(pending) - 1: delta = reward + config.discount * R - value.data else: - delta = reward + pending[i + 1][2].data - value.data + delta = reward + config.discount * pending[i + 1][2].data - value.data GAE = config.discount * config.gae_tau * GAE + delta loss += -log_prob.gather(1, Variable(torch.LongTensor([[action]]))) * Variable(GAE) loss += config.entropy_weight * torch.sum(torch.mul(prob, log_prob)) @@ -68,11 +69,7 @@ class AdvantageActorCritic: self.optimizer.zero_grad() loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) - for param, worker_param in zip( - self.learning_network.parameters(), self.worker_network.parameters()): - if param.grad is not None: - break - param._grad = worker_param.grad + sync_grad(self.learning_network, self.worker_network) self.optimizer.step() self.worker_network.load_state_dict(self.learning_network.state_dict()) self.worker_network.reset(terminal) diff --git a/async_worker/continuous_actor_critic.py b/async_worker/continuous_actor_critic.py index dabb0dd..31b8abb 100644 --- a/async_worker/continuous_actor_critic.py +++ b/async_worker/continuous_actor_critic.py @@ -92,11 +92,7 @@ class ContinuousAdvantageActorCritic: actor_loss.backward() critic_loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) - for param, worker_param in zip( - self.learning_network.parameters(), self.worker_network.parameters()): - if param.grad is not None: - break - param._grad = worker_param.grad + sync_grad(self.learning_network, self.worker_network) self.actor_opt.step() self.critic_opt.step() self.worker_network.load_state_dict(self.learning_network.state_dict()) diff --git a/agent/DDPG_agent.py b/async_worker/dpg.py similarity index 59% rename from agent/DDPG_agent.py rename to async_worker/dpg.py index 600c334..6137c25 100644 --- a/agent/DDPG_agent.py +++ b/async_worker/dpg.py @@ -4,31 +4,35 @@ # declaration at the top # ####################################################################### +import numpy as np +import torch.multiprocessing as mp from network import * -from component import * from utils import * +from component import * +from async_worker import * import pickle -import torch.nn as nn +import os +import time -class DDPGAgent: - def __init__(self, config): +class DeterministicPolicyGradient: + def __init__(self, config, shared_network, extra): self.config = config self.task = config.task_fn() - self.learning_network = config.network_fn() + + self.shared_network = shared_network + self.worker_network = config.network_fn() + self.worker_network.load_state_dict(self.shared_network.state_dict()) self.target_network = config.network_fn() - self.target_network.load_state_dict(self.learning_network.state_dict()) - self.target_network.eval() - self.actor_opt = config.actor_optimizer_fn(self.learning_network.actor.parameters()) - self.critic_opt = config.critic_optimizer_fn(self.learning_network.critic.parameters()) - self.replay = config.replay_fn() + self.target_network.load_state_dict(self.worker_network.state_dict()) + self.actor_opt = config.actor_optimizer_fn(self.shared_network.actor.parameters()) + self.critic_opt = config.critic_optimizer_fn(self.shared_network.critic.parameters()) + self.random_process = config.random_process_fn() self.criterion = nn.MSELoss() - self.total_steps = 0 - self.epsilon = 1.0 - self.d_epsilon = 1.0 / config.noise_decay_interval - self.state_normalizer = Normalizer(self.task.state_dim) - self.reward_normalizer = Normalizer(1) + self.shared_state_normalizer, self.shared_reward_normalizer, self.replay = extra + self.state_normalizer = StaticNormalizer(self.task.state_dim) + self.reward_normalizer = StaticNormalizer(1) def soft_update(self, target, src): for target_param, param in zip(target.parameters(), src.parameters()): @@ -41,8 +45,8 @@ class DDPGAgent: state = self.state_normalizer(state) config = self.config - actor = self.learning_network.actor - critic = self.learning_network.critic + actor = self.worker_network.actor + critic = self.worker_network.critic target_actor = self.target_network.actor target_critic = self.target_network.critic @@ -52,11 +56,7 @@ class DDPGAgent: actor.eval() action = actor.predict(np.stack([state])).flatten() if not deterministic: - if self.total_steps < config.exploration_steps: - action = self.task.random_action() - else: - action += max(self.epsilon, config.min_epsilon) * self.random_process.sample() - self.epsilon -= self.d_epsilon + action += self.random_process.sample() next_state, reward, done, info = self.task.step(action) done = (done or (config.max_episode_length and steps >= config.max_episode_length)) next_state = self.state_normalizer(next_state) @@ -65,15 +65,17 @@ class DDPGAgent: if not deterministic: self.replay.feed([state, action, reward, next_state, int(done)]) - self.total_steps += 1 + with config.steps_lock: + config.total_steps.value += 1 + steps += 1 state = next_state if done: break - if not deterministic and self.total_steps > config.exploration_steps: - self.learning_network.train() + if not deterministic and self.replay.size() >= config.min_memory_size: + self.worker_network.train() experiences = self.replay.sample() states, actions, rewards, next_states, terminals = experiences q_next = target_critic.predict(next_states, target_actor.predict(next_states)) @@ -86,8 +88,11 @@ class DDPGAgent: critic_loss = self.criterion(q, q_next) critic.zero_grad() + self.critic_opt.zero_grad() critic_loss.backward() - self.critic_opt.step() + with config.network_lock: + sync_grad(self.shared_network.critic, critic) + self.critic_opt.step() actions = actor.predict(states, False) var_actions = Variable(actions.data, requires_grad=True) @@ -95,13 +100,20 @@ class DDPGAgent: q.backward(torch.ones(q.size())) actor.zero_grad() + self.actor_opt.zero_grad() actions.backward(-var_actions.grad.data) - self.actor_opt.step() + with config.network_lock: + sync_grad(self.shared_network.actor, actor) + self.actor_opt.step() - self.soft_update(self.target_network, self.learning_network) + self.worker_network.load_state_dict(self.shared_network.state_dict()) - return total_reward, steps + self.soft_update(self.target_network, self.worker_network) - def save(self, file_name): - with open(file_name, 'wb') as f: - pickle.dump(self.learning_network.state_dict(), f) + self.shared_state_normalizer.offline_stats.merge(self.state_normalizer.online_stats) + self.state_normalizer.online_stats.zero() + + self.shared_reward_normalizer.offline_stats.merge(self.reward_normalizer.online_stats) + self.reward_normalizer.online_stats.zero() + + return steps, total_reward diff --git a/async_worker/n_step_q.py b/async_worker/n_step_q.py index 3a858cf..3a5899b 100644 --- a/async_worker/n_step_q.py +++ b/async_worker/n_step_q.py @@ -7,6 +7,7 @@ import numpy as np import torch from torch.autograd import Variable import torch.nn as nn +from utils import * class NStepQLearning: def __init__(self, config, learning_network, target_network): @@ -63,11 +64,7 @@ class NStepQLearning: self.optimizer.zero_grad() loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) - for param, worker_param in zip( - self.learning_network.parameters(), self.worker_network.parameters()): - if param.grad is not None: - break - param._grad = worker_param.grad + sync_grad(self.learning_network, self.worker_network) self.optimizer.step() self.worker_network.load_state_dict(self.learning_network.state_dict()) self.worker_network.reset(terminal) diff --git a/async_worker/one_step_q.py b/async_worker/one_step_q.py index 9f1213e..d190753 100644 --- a/async_worker/one_step_q.py +++ b/async_worker/one_step_q.py @@ -7,6 +7,7 @@ import numpy as np import torch from torch.autograd import Variable import torch.nn as nn +from utils import * class OneStepQLearning: def __init__(self, config, learning_network, target_network): @@ -60,11 +61,7 @@ class OneStepQLearning: self.optimizer.zero_grad() loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) - for param, worker_param in zip( - self.learning_network.parameters(), self.worker_network.parameters()): - if param.grad is not None: - break - param._grad = worker_param.grad + sync_grad(self.learning_network, self.worker_network) self.optimizer.step() self.worker_network.load_state_dict(self.learning_network.state_dict()) self.worker_network.reset(terminal) diff --git a/async_worker/one_step_sarsa.py b/async_worker/one_step_sarsa.py index b5123e9..d763867 100644 --- a/async_worker/one_step_sarsa.py +++ b/async_worker/one_step_sarsa.py @@ -7,6 +7,7 @@ import numpy as np import torch from torch.autograd import Variable import torch.nn as nn +from utils import * class OneStepSarsa: def __init__(self, config, learning_network, target_network): @@ -65,11 +66,7 @@ class OneStepSarsa: self.optimizer.zero_grad() loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) - for param, worker_param in zip( - self.learning_network.parameters(), self.worker_network.parameters()): - if param.grad is not None: - break - param._grad = worker_param.grad + sync_grad(self.learning_network, self.worker_network) self.optimizer.step() self.worker_network.load_state_dict(self.learning_network.state_dict()) self.worker_network.reset(terminal) diff --git a/async_worker/ppo.py b/async_worker/ppo.py index 9224c93..58f1ea8 100644 --- a/async_worker/ppo.py +++ b/async_worker/ppo.py @@ -72,7 +72,6 @@ class ProximalPolicyOptimization: values.append(value) state, reward, done, _ = self.task.step(action) state = self.state_normalizer(state) - # print state done = (done or (config.max_episode_length and episode_length > config.max_episode_length)) batched_rewards += reward @@ -153,8 +152,7 @@ class ProximalPolicyOptimization: self.shared_network.zero_grad() self.actor_opt.zero_grad() self.critic_opt.zero_grad() - for param, worker_param in zip(self.shared_network.parameters(), self.worker_network.parameters()): - param._grad = worker_param.grad.clone() + sync_grad(self.shared_network, self.worker_network) self.actor_opt.step() self.critic_opt.step() diff --git a/component/random_process.py b/component/random_process.py index 70caf4a..6e8d252 100644 --- a/component/random_process.py +++ b/component/random_process.py @@ -46,4 +46,4 @@ class OrnsteinUhlenbeckProcess(AnnealedGaussianProcess): return x def reset_states(self): - self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size) \ No newline at end of file + self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size) diff --git a/component/replay.py b/component/replay.py index ff07329..48fb2d0 100644 --- a/component/replay.py +++ b/component/replay.py @@ -7,6 +7,7 @@ import numpy as np import torch import random +import torch.multiprocessing as mp class Replay: def __init__(self, memory_size, batch_size, dtype=np.float32): @@ -95,6 +96,62 @@ class HybridRewardReplay: self.next_states[sampled_indices], self.terminals[sampled_indices]] +class SharedReplay: + def __init__(self, memory_size, batch_size, state_shape, action_shape): + self.memory_size = memory_size + self.batch_size = batch_size + + self.states = torch.zeros((self.memory_size, ) + state_shape) + self.actions = torch.zeros((self.memory_size, ) + action_shape) + self.rewards = torch.zeros(self.memory_size) + self.next_states = torch.zeros((self.memory_size, ) + state_shape) + self.terminals = torch.zeros(self.memory_size) + + self.states.share_memory_() + self.actions.share_memory_() + self.rewards.share_memory_() + self.next_states.share_memory_() + self.terminals.share_memory_() + + self.pos = 0 + self.full = False + self.buffer_lock = mp.Lock() + + def feed_(self, experience): + state, action, reward, next_state, done = experience + self.states[self.pos][:] = torch.FloatTensor(state) + self.actions[self.pos][:] = torch.FloatTensor(action) + self.rewards[self.pos] = reward + self.next_states[self.pos][:] = torch.FloatTensor(next_state) + self.terminals[self.pos] = done + + self.pos += 1 + if self.pos == self.memory_size: + self.full = True + self.pos = 0 + + def size(self): + if self.full: + return self.memory_size + return self.pos + + def sample_(self): + upper_bound = self.memory_size if self.full else self.pos + sampled_indices = torch.LongTensor(np.random.randint(0, upper_bound, size=self.batch_size)) + return [self.states[sampled_indices], + self.actions[sampled_indices], + self.rewards[sampled_indices], + self.next_states[sampled_indices], + self.terminals[sampled_indices]] + + def feed(self, experience): + with self.buffer_lock: + self.feed_(experience) + + def sample(self): + with self.buffer_lock: + return self.sample_() + class HighDimActionReplay: def __init__(self, memory_size, batch_size, dtype=np.float32): self.memory_size = memory_size @@ -130,6 +187,11 @@ class HighDimActionReplay: self.full = True self.pos = 0 + def size(self): + if self.full: + return self.memory_size + return self.pos + def sample(self): upper_bound = self.memory_size if self.full else self.pos sampled_indices = np.random.randint(0, upper_bound, size=self.batch_size) diff --git a/component/task.py b/component/task.py index 31e7611..d52d009 100644 --- a/component/task.py +++ b/component/task.py @@ -72,6 +72,7 @@ class PixelAtari(BasicTask): env = FireResetEnv(env) env = ProcessFrame(env, frame_size) self.env = ClippedRewardsWrapper(env) + self.action_dim = self.env.action_space.n def normalize_state(self, state): return np.asarray(state, dtype=np.float32) / 255.0 diff --git a/images/DDPG-RoboschoolReacher-v1.png b/images/DDPG-RoboschoolReacher-v1.png deleted file mode 100644 index 08d937c..0000000 Binary files a/images/DDPG-RoboschoolReacher-v1.png and /dev/null differ diff --git a/images/DDPG.png b/images/DDPG.png index 2841c5d..164511c 100644 Binary files a/images/DDPG.png and b/images/DDPG.png differ diff --git a/images/DPPO.png b/images/DPPO.png deleted file mode 100644 index ec2a387..0000000 Binary files a/images/DPPO.png and /dev/null differ diff --git a/images/P3O.png b/images/P3O.png new file mode 100644 index 0000000..7d21f7d Binary files /dev/null and b/images/P3O.png differ diff --git a/main.py b/main.py index ddae220..6e2ca81 100644 --- a/main.py +++ b/main.py @@ -65,10 +65,10 @@ def a3c_cart_pole(): def dqn_pixel_atari(name): config = Config() config.history_length = 4 - n_actions = 6 config.task_fn = lambda: PixelAtari(name, no_op=30, frame_skip=4, normalized_state=False) + action_dim = config.task_fn().action_dim config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) - config.network_fn = lambda optimizer_fn: NatureConvNet(config.history_length, n_actions, optimizer_fn) + config.network_fn = lambda optimizer_fn: NatureConvNet(config.history_length, action_dim, optimizer_fn) # config.network_fn = lambda optimizer_fn: DuelingNatureConvNet(config.history_length, n_actions, optimizer_fn) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8) @@ -199,11 +199,13 @@ def a3c_continuous(): agent = AsyncAgent(config) agent.run() -def dppo_continuous(): +def p3o_continuous(): config = Config() config.task_fn = lambda: Pendulum() + # config.task_fn = lambda: BipedalWalker() # config.task_fn = lambda: BipedalWalkerHardcore() # config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1') + # config.task_fn = lambda: Roboschool('RoboschoolAnt-v1') task = config.task_fn() config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim, gpu=False, unit_std=True) @@ -217,7 +219,7 @@ def dppo_continuous(): config.worker = ProximalPolicyOptimization config.discount = 0.99 config.gae_tau = 0.97 - config.num_workers = 8 + config.num_workers = 6 config.test_interval = 1 config.test_repetitions = 1 config.max_episode_length = task.max_episode_steps @@ -230,11 +232,13 @@ def dppo_continuous(): agent = AsyncAgent(config) agent.run() -def ddpg_continuous(): +def d3pg_continuous(): config = Config() - config.task_fn = lambda: Pendulum() + # config.task_fn = lambda: Pendulum() + # config.task_fn = lambda: ContinuousLunarLander() # config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1') - # config.task_fn = lambda: Roboschool('RoboschoolReacher-v1') + config.task_fn = lambda: Roboschool('RoboschoolReacher-v1') + # config.task_fn = lambda: BipedalWalker() task = config.task_fn() config.actor_network_fn = lambda: DeterministicActorNet( task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.relu, batch_norm=False) @@ -243,32 +247,35 @@ def ddpg_continuous(): config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) config.critic_optimizer_fn =\ - lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) - config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) + lambda params: torch.optim.Adam(params, lr=1e-4) + config.replay_fn = lambda: SharedReplay(memory_size=1000000, batch_size=64, + state_shape=(task.state_dim, ), action_shape=(task.action_dim, )) config.discount = 0.99 config.max_episode_length = task.max_episode_steps - config.target_network_mix = 0.001 - config.exploration_steps = 100 - config.noise_decay_interval = 10000 - config.min_epsilon = 0.1 config.random_process_fn = \ - lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) - config.test_interval = 0 - config.test_repetitions = 10 - config.save_interval = 50 + lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2, + n_steps_annealing=100000) + config.worker = DeterministicPolicyGradient + config.num_workers = 6 + config.min_memory_size = 50 + config.target_network_mix = 0.001 + config.test_interval = 500 + config.test_repetitions = 1 + config.gradient_clip = 20 config.logger = Logger('./log', gym.logger) - run_episodes(DDPGAgent(config)) + agent = AsyncAgent(config) + agent.run() if __name__ == '__main__': # gym.logger.setLevel(logging.DEBUG) gym.logger.setLevel(logging.INFO) - dqn_cart_pole() + # dqn_cart_pole() # async_cart_pole() # a3c_cart_pole() # a3c_continuous() - # dppo_continuous() - # ddpg_continuous() + # p3o_continuous() + d3pg_continuous() # dqn_fruit() # hrdqn_fruit() diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index a77281c..d900094 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -154,7 +154,7 @@ class GaussianActorNet(nn.Module, BasicNet): def log_density(self, x, mean, log_std, std): var = std.pow(2) - log_density = -(x - mean).pow(2) / (2 * var) - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x)) - log_std + log_density = -(x - mean).pow(2) / (2 * var + 1e-5) - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x)) - log_std return log_density.sum(1) def entropy(self, std): diff --git a/utils/__init__.py b/utils/__init__.py index ba7d6a7..c9b2200 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -1,6 +1,6 @@ from .config import * from .normalizer import * -from .run import * +from .misc import * try: from .tf_logger import Logger diff --git a/utils/config.py b/utils/config.py index bad7273..132a866 100644 --- a/utils/config.py +++ b/utils/config.py @@ -48,3 +48,4 @@ class Config: self.min_epsilon = 0 self.save_interval = 0 self.max_steps = 0 + self.success_threshold = float('inf') diff --git a/utils/run.py b/utils/misc.py similarity index 90% rename from utils/run.py rename to utils/misc.py index 3976404..8f4ba1a 100644 --- a/utils/run.py +++ b/utils/misc.py @@ -50,4 +50,8 @@ def run_episodes(agent): if avg_reward > agent.task.success_threshold: break - return steps, rewards, avg_test_rewards \ No newline at end of file + return steps, rewards, avg_test_rewards + +def sync_grad(target_network, src_network): + for param, src_param in zip(target_network.parameters(), src_network.parameters()): + param._grad = src_param.grad.clone() \ No newline at end of file