diff --git a/README.md b/README.md index 71876df..8d4cfa3 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Implemented algorithms: * Distributed Proximal Policy Optimization (DPPO) # Curves -> Curves for CartPole are trivial so I didn't place it here. +> Curves for CartPole are trivial so I didn't place it here. There isn't any fixed random seed. ## DQN, Double DQN, Dueling DQN ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DQN-breakout.png) ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DQN-Pong.png) @@ -51,7 +51,8 @@ variance unbounded, which is also included in the implementation. ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DDPG-Pendulum-v0.png) -Current DDPG implementation seems to have potential bugs, I'm now actively working on it. +Extra caution is necessary when computing gradients, the [repo](https://github.com/ghliu/pytorch-ddpg) I referred +seems to have critical bugs. Anyway DDPG is fairly unstable. ## DPPO diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 1c5619b..8b86e4c 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -89,11 +89,13 @@ class DDPGAgent: critic_loss.backward() self.critic_opt.step() - actor_loss = -critic.predict(states, actor.predict(states, False)) - actor_loss = actor_loss.mean() + actions = actor.predict(states, False) + var_actions = Variable(actions.data, requires_grad=True) + q = critic.predict(states, var_actions) + q.backward(torch.ones(q.size())) actor.zero_grad() - actor_loss.backward() + actions.backward(-var_actions.grad.data) self.actor_opt.step() self.soft_update(self.target_network, self.learning_network) diff --git a/component/task.py b/component/task.py index a6eb803..6d46885 100644 --- a/component/task.py +++ b/component/task.py @@ -76,9 +76,28 @@ class PixelAtari(BasicTask): def normalize_state(self, state): return np.asarray(state, dtype=np.float32) / 255.0 +class ContinuousMountainCar(BasicTask): + name = 'MountainCarContinuous-v0' + success_threshold = 90 + default_max_episode = 999 + + def __init__(self): + BasicTask.__init__(self) + self.env = gym.make(self.name) + self.env._max_episode_steps = sys.maxsize + self.action_dim = self.env.action_space.shape[0] + self.state_dim = self.env.observation_space.shape[0] + + def step(self, action): + action = np.clip(action, -1, 1) + next_state, reward, done, info = self.env.step(action) + return next_state, reward, done, info + + class Pendulum(BasicTask): name = 'Pendulum-v0' success_threshold = -10 + default_max_episode = 200 def __init__(self): BasicTask.__init__(self) @@ -95,6 +114,24 @@ class Pendulum(BasicTask): class BipedalWalker(BasicTask): name = 'BipedalWalker-v2' success_threshold = 300 + default_max_episode = 999 + + def __init__(self): + BasicTask.__init__(self) + self.env = gym.make(self.name) + self.env._max_episode_steps = sys.maxsize + self.action_dim = self.env.action_space.shape[0] + self.state_dim = self.env.observation_space.shape[0] + + def step(self, action): + action = np.clip(action, -1, 1) + next_state, reward, done, info = self.env.step(action) + return next_state, reward, done, info + +class BipedalWalkerHardcore(BasicTask): + name = 'BipedalWalkerHardcore-v2' + success_threshold = 300 + default_max_episode = 2000 def __init__(self): BasicTask.__init__(self) @@ -111,6 +148,7 @@ class BipedalWalker(BasicTask): class ContinuousLunarLander(BasicTask): name = 'LunarLanderContinuous-v2' success_threshold = 300 + default_max_episode = 1000 def __init__(self): BasicTask.__init__(self) diff --git a/images/DDPG-Pendulum-v0.png b/images/DDPG-Pendulum-v0.png index 4da5d1b..09b9296 100644 Binary files a/images/DDPG-Pendulum-v0.png and b/images/DDPG-Pendulum-v0.png differ diff --git a/main.py b/main.py index 3f84aad..0b94f47 100644 --- a/main.py +++ b/main.py @@ -62,75 +62,6 @@ def a3c_cart_pole(): agent = AsyncAgent(config) agent.run() -def a3c_pendulum(): - config = Config() - config.task_fn = lambda: Pendulum() - task = config.task_fn() - config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001) - config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) - config.network_fn = lambda: DisjointActorCriticNet( - lambda: GaussianActorNet(task.state_dim, task.action_dim), - lambda: GaussianCriticNet(task.state_dim)) - config.policy_fn = lambda: GaussianPolicy() - config.worker = ContinuousAdvantageActorCritic - config.discount = 0.99 - config.max_episode_length = 200 - config.num_workers = 8 - config.update_interval = 5 - config.test_interval = 1 - config.test_repetitions = 5 - config.entropy_weight = 0 - config.gradient_clip = 40 - config.logger = Logger('./log', gym.logger) - agent = AsyncAgent(config) - agent.run() - -def a3c_lunar_lander(): - config = Config() - config.task_fn = lambda: ContinuousLunarLander() - task = config.task_fn() - config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001) - config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) - config.network_fn = lambda: DisjointActorCriticNet( - lambda: GaussianActorNet(task.state_dim, task.action_dim), - lambda: GaussianCriticNet(task.state_dim)) - config.policy_fn = lambda: GaussianPolicy() - config.worker = ContinuousAdvantageActorCritic - config.discount = 0.99 - config.max_episode_length = 1000 - config.num_workers = 8 - config.update_interval = 5 - config.test_interval = 1 - config.test_repetitions = 5 - config.entropy_weight = 0 - config.gradient_clip = 40 - config.logger = Logger('./log', gym.logger) - agent = AsyncAgent(config) - agent.run() - -def a3c_walker(): - config = Config() - config.task_fn = lambda: BipedalWalker() - task = config.task_fn() - config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001) - config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) - config.network_fn = lambda: DisjointActorCriticNet( - lambda: GaussianActorNet(task.state_dim, task.action_dim), - lambda: GaussianCriticNet(task.state_dim)) - config.policy_fn = lambda: GaussianPolicy() - config.worker = ContinuousAdvantageActorCritic - config.discount = 0.99 - config.max_episode_length = 999 - config.num_workers = 8 - config.update_interval = 20 - config.test_interval = 1 - config.test_repetitions = 5 - config.entropy_weight = 0 - config.gradient_clip = 40 - config.logger = Logger('./log', gym.logger) - agent = AsyncAgent(config) - agent.run() - def dqn_pixel_atari(name): config = Config() config.history_length = 4 @@ -197,88 +128,6 @@ def a3c_pixel_atari(name): agent = AsyncAgent(config) agent.run() -def ddpg_pendulum(): - task_fn = lambda: Pendulum() - task = task_fn() - config = Config() - config.task_fn = task_fn - config.actor_network_fn = lambda: DeterministicActorNet( - task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.relu, batch_norm=False) - config.critic_network_fn = lambda: DeterministicCriticNet( - task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False) - config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) - config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) - config.critic_optimizer_fn =\ - lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) - config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) - config.discount = 0.99 - config.max_episode_length = 200 - config.target_network_mix = 0.001 - config.exploration_steps = 100 - config.noise_decay_interval = 10000 - config.random_process_fn = \ - lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) - config.test_interval = 0 - config.test_repetitions = 10 - config.save_interval = 50 - config.logger = Logger('./log', gym.logger) - run_episodes(DDPGAgent(config)) - -def ddpg_lunar_lander(): - task_fn = lambda: ContinuousLunarLander() - task = task_fn() - config = Config() - config.task_fn = task_fn - config.actor_network_fn = lambda: DeterministicActorNet( - task.state_dim, task.action_dim, F.tanh, 1, batch_norm=True) - config.critic_network_fn = lambda: DeterministicCriticNet( - task.state_dim, task.action_dim, batch_norm=True) - config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) - config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) - config.critic_optimizer_fn =\ - lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) - config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) - config.discount = 0.99 - config.max_episode_length = 1000 - config.target_network_mix = 0.001 - config.exploration_steps = 100 - config.noise_decay_interval = 10000 - config.random_process_fn = \ - lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) - config.test_interval = 0 - config.test_repetitions = 10 - config.save_interval = 50 - config.logger = Logger('./log', gym.logger) - run_episodes(DDPGAgent(config)) - -def ddpg_walker(): - task_fn = lambda: BipedalWalker() - task = task_fn() - config = Config() - config.task_fn = task_fn - config.actor_network_fn = lambda: DeterministicActorNet( - task.state_dim, task.action_dim, F.tanh, 1, gpu=True, batch_norm=False, non_linear=F.tanh) - config.critic_network_fn = lambda: DeterministicCriticNet( - task.state_dim, task.action_dim, gpu=True, batch_norm=False, non_linear=F.tanh) - config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) - config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) - config.critic_optimizer_fn =\ - lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) - config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) - config.discount = 0.99 - config.min_epsilon = 0.1 - config.max_episode_length = 999 - config.target_network_mix = 0.001 - config.exploration_steps = 10000 - config.noise_decay_interval = 1000000 - config.random_process_fn = \ - lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) - config.test_interval = 0 - config.test_repetitions = 5 - config.save_interval = 50 - config.logger = Logger('./log', gym.logger) - run_episodes(DDPGAgent(config)) - def dqn_fruit(): config = Config() config.task_fn = lambda: Fruit() @@ -350,11 +199,38 @@ def hrmsdqn_fruit(): config.episode_limit = 5000 run_episodes(MSDQNAgent(config)) -def ppo_pendulum(): +def a3c_continuous(): config = Config() - config.task_fn = lambda: Pendulum() + # config.task_fn = lambda: Pendulum() + config.task_fn = lambda: BipedalWalkerHardcore() task = config.task_fn() - config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim, gpu=False) + config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001) + config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) + config.network_fn = lambda: DisjointActorCriticNet( + # lambda: GaussianActorNet(task.state_dim, task.action_dim, unit_std=False, action_gate=F.tanh, action_scale=2.0), + lambda: GaussianActorNet(task.state_dim, task.action_dim, unit_std=True), + lambda: GaussianCriticNet(task.state_dim)) + config.policy_fn = lambda: GaussianPolicy() + config.worker = ContinuousAdvantageActorCritic + config.discount = 0.99 + config.max_episode_length = task.default_max_episode + config.num_workers = 8 + config.update_interval = 20 + config.test_interval = 1 + config.test_repetitions = 1 + config.entropy_weight = 0 + config.gradient_clip = 40 + config.logger = Logger('./log', gym.logger) + agent = AsyncAgent(config) + agent.run() + +def dppo_continuous(): + config = Config() + # config.task_fn = lambda: Pendulum() + config.task_fn = lambda: BipedalWalkerHardcore() + task = config.task_fn() + config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim, + gpu=False, unit_std=True) config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim, gpu=False) config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) @@ -368,7 +244,7 @@ def ppo_pendulum(): config.num_workers = 8 config.test_interval = 1 config.test_repetitions = 1 - config.max_episode_length = 200 + config.max_episode_length = task.default_max_episode config.entropy_weight = 0 config.gradient_clip = 40 config.rollout_length = 10000 @@ -378,61 +254,32 @@ def ppo_pendulum(): agent = AsyncAgent(config) agent.run() -def ppo_lunar_lander(): +def ddpg_continuous(): + task_fn = lambda: Pendulum() + task = task_fn() config = Config() - config.task_fn = lambda: ContinuousLunarLander() - task = config.task_fn() - config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim, gpu=False) - config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim, gpu=False) + config.task_fn = task_fn + config.actor_network_fn = lambda: DeterministicActorNet( + task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.relu, batch_norm=False) + config.critic_network_fn = lambda: DeterministicCriticNet( + task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False) config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) - config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) - config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) - - config.policy_fn = lambda: GaussianPolicy() - config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=2048) - config.worker = ProximalPolicyOptimization + config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) + config.critic_optimizer_fn =\ + lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) + config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) config.discount = 0.99 - config.gae_tau = 0.97 - config.num_workers = 8 - config.test_interval = 1 - config.test_repetitions = 1 - config.max_episode_length = 1000 - config.entropy_weight = 0 - config.gradient_clip = 40 - config.rollout_length = 10000 - config.optimize_epochs = 1 - config.ppo_ratio_clip = 0.2 + config.max_episode_length = task.default_max_episode + config.target_network_mix = 0.001 + config.exploration_steps = 100 + config.noise_decay_interval = 10000 + config.random_process_fn = \ + lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) + config.test_interval = 0 + config.test_repetitions = 10 + config.save_interval = 50 config.logger = Logger('./log', gym.logger) - agent = AsyncAgent(config) - agent.run() - -def ppo_walker(): - config = Config() - config.task_fn = lambda: BipedalWalker() - task = config.task_fn() - config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim, gpu=False) - config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim, gpu=False) - config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) - config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) - config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) - - config.policy_fn = lambda: GaussianPolicy() - config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=2048) - config.worker = ProximalPolicyOptimization - config.discount = 0.99 - config.gae_tau = 0.97 - config.num_workers = 8 - config.test_interval = 1 - config.test_repetitions = 1 - config.max_episode_length = 999 - config.entropy_weight = 0 - config.gradient_clip = 40 - config.rollout_length = 10000 - config.optimize_epochs = 1 - config.ppo_ratio_clip = 0.2 - config.logger = Logger('./log', gym.logger) - agent = AsyncAgent(config) - agent.run() + run_episodes(DDPGAgent(config)) if __name__ == '__main__': # gym.logger.setLevel(logging.DEBUG) @@ -441,15 +288,9 @@ if __name__ == '__main__': # dqn_cart_pole() # async_cart_pole() # a3c_cart_pole() - # a3c_pendulum() - # a3c_lunar_lander() - # a3c_walker() - # ddpg_pendulum() - # ddpg_lunar_lander() - # ddpg_walker() - # ppo_pendulum() - # ppo_lunar_lander() - ppo_walker() + # a3c_continuous() + # dppo_continuous() + ddpg_continuous() # dqn_fruit() # hrdqn_fruit() diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index ff53b77..83ad11a 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -145,7 +145,7 @@ class GaussianActorNet(nn.Module, BasicNet): log_std = self.action_log_std.expand_as(mean) std = log_std.exp() else: - std = F.softplus(self.fc_std(x) + 1e-5) + std = F.softplus(self.action_std(phi) + 1e-5) log_std = std.log() return mean, std, log_std