Fix a critical bug in DDPG

2026-09-09 11:13:47 +08:00 · 2017-10-15 10:05:33 -06:00
parent 3ed6f2a9db
commit 1b6de16b4f
6 changed files with 103 additions and 221 deletions
@@ -17,7 +17,7 @@ Implemented algorithms:
 * Distributed Proximal Policy Optimization (DPPO)

 # Curves
-> Curves for CartPole are trivial so I didn't place it here.
+> Curves for CartPole are trivial so I didn't place it here. There isn't any fixed random seed.
 ## DQN, Double DQN, Dueling DQN 
 ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DQN-breakout.png)
 ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DQN-Pong.png)
@@ -51,7 +51,8 @@ variance unbounded, which is also included in the implementation.

 ![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DDPG-Pendulum-v0.png)

-Current DDPG implementation seems to have potential bugs, I'm now actively working on it.
+Extra caution is necessary when computing gradients, the [repo](https://github.com/ghliu/pytorch-ddpg) I referred
+seems to have critical bugs. Anyway DDPG is fairly unstable.

 ## DPPO

@@ -89,11 +89,13 @@ class DDPGAgent:
                critic_loss.backward()
                self.critic_opt.step()

-                actor_loss = -critic.predict(states, actor.predict(states, False))
-                actor_loss = actor_loss.mean()
+                actions = actor.predict(states, False)
+                var_actions = Variable(actions.data, requires_grad=True)
+                q = critic.predict(states, var_actions)
+                q.backward(torch.ones(q.size()))

                actor.zero_grad()
-                actor_loss.backward()
+                actions.backward(-var_actions.grad.data)
                self.actor_opt.step()

                self.soft_update(self.target_network, self.learning_network)
@@ -76,9 +76,28 @@ class PixelAtari(BasicTask):
    def normalize_state(self, state):
        return np.asarray(state, dtype=np.float32) / 255.0

+class ContinuousMountainCar(BasicTask):
+    name = 'MountainCarContinuous-v0'
+    success_threshold = 90
+    default_max_episode = 999
+
+    def __init__(self):
+        BasicTask.__init__(self)
+        self.env = gym.make(self.name)
+        self.env._max_episode_steps = sys.maxsize
+        self.action_dim = self.env.action_space.shape[0]
+        self.state_dim = self.env.observation_space.shape[0]
+
+    def step(self, action):
+        action = np.clip(action, -1, 1)
+        next_state, reward, done, info = self.env.step(action)
+        return next_state, reward, done, info
+
+
 class Pendulum(BasicTask):
    name = 'Pendulum-v0'
    success_threshold = -10
+    default_max_episode = 200

    def __init__(self):
        BasicTask.__init__(self)
@@ -95,6 +114,24 @@ class Pendulum(BasicTask):
 class BipedalWalker(BasicTask):
    name = 'BipedalWalker-v2'
    success_threshold = 300
+    default_max_episode = 999
+
+    def __init__(self):
+        BasicTask.__init__(self)
+        self.env = gym.make(self.name)
+        self.env._max_episode_steps = sys.maxsize
+        self.action_dim = self.env.action_space.shape[0]
+        self.state_dim = self.env.observation_space.shape[0]
+
+    def step(self, action):
+        action = np.clip(action, -1, 1)
+        next_state, reward, done, info = self.env.step(action)
+        return next_state, reward, done, info
+
+class BipedalWalkerHardcore(BasicTask):
+    name = 'BipedalWalkerHardcore-v2'
+    success_threshold = 300
+    default_max_episode = 2000

    def __init__(self):
        BasicTask.__init__(self)
@@ -111,6 +148,7 @@ class BipedalWalker(BasicTask):
 class ContinuousLunarLander(BasicTask):
    name = 'LunarLanderContinuous-v2'
    success_threshold = 300
+    default_max_episode = 1000

    def __init__(self):
        BasicTask.__init__(self)
@@ -62,75 +62,6 @@ def a3c_cart_pole():
    agent = AsyncAgent(config)
    agent.run()

-def a3c_pendulum():
-    config = Config()
-    config.task_fn = lambda: Pendulum()
-    task = config.task_fn()
-    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001)
-    config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
-    config.network_fn = lambda: DisjointActorCriticNet(
-        lambda: GaussianActorNet(task.state_dim, task.action_dim),
-        lambda: GaussianCriticNet(task.state_dim))
-    config.policy_fn = lambda: GaussianPolicy()
-    config.worker = ContinuousAdvantageActorCritic
-    config.discount = 0.99
-    config.max_episode_length = 200
-    config.num_workers = 8
-    config.update_interval = 5
-    config.test_interval = 1
-    config.test_repetitions = 5
-    config.entropy_weight = 0
-    config.gradient_clip = 40
-    config.logger = Logger('./log', gym.logger)
-    agent = AsyncAgent(config)
-    agent.run()
-
-def a3c_lunar_lander():
-    config = Config()
-    config.task_fn = lambda: ContinuousLunarLander()
-    task = config.task_fn()
-    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001)
-    config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
-    config.network_fn = lambda: DisjointActorCriticNet(
-        lambda: GaussianActorNet(task.state_dim, task.action_dim),
-        lambda: GaussianCriticNet(task.state_dim))
-    config.policy_fn = lambda: GaussianPolicy()
-    config.worker = ContinuousAdvantageActorCritic
-    config.discount = 0.99
-    config.max_episode_length = 1000
-    config.num_workers = 8
-    config.update_interval = 5
-    config.test_interval = 1
-    config.test_repetitions = 5
-    config.entropy_weight = 0
-    config.gradient_clip = 40
-    config.logger = Logger('./log', gym.logger)
-    agent = AsyncAgent(config)
-    agent.run()
-
-def a3c_walker():
-    config = Config()
-    config.task_fn = lambda: BipedalWalker()
-    task = config.task_fn()
-    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001)
-    config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
-    config.network_fn = lambda: DisjointActorCriticNet(
-        lambda: GaussianActorNet(task.state_dim, task.action_dim),
-        lambda: GaussianCriticNet(task.state_dim))
-    config.policy_fn = lambda: GaussianPolicy()
-    config.worker = ContinuousAdvantageActorCritic
-    config.discount = 0.99
-    config.max_episode_length = 999
-    config.num_workers = 8
-    config.update_interval = 20
-    config.test_interval = 1
-    config.test_repetitions = 5
-    config.entropy_weight = 0
-    config.gradient_clip = 40
-    config.logger = Logger('./log', gym.logger)
-    agent = AsyncAgent(config)
-    agent.run()
-
 def dqn_pixel_atari(name):
    config = Config()
    config.history_length = 4
@@ -197,88 +128,6 @@ def a3c_pixel_atari(name):
    agent = AsyncAgent(config)
    agent.run()

-def ddpg_pendulum():
-    task_fn = lambda: Pendulum()
-    task = task_fn()
-    config = Config()
-    config.task_fn = task_fn
-    config.actor_network_fn = lambda: DeterministicActorNet(
-        task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.relu, batch_norm=False)
-    config.critic_network_fn = lambda: DeterministicCriticNet(
-        task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False)
-    config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
-    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
-    config.critic_optimizer_fn =\
-        lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
-    config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
-    config.discount = 0.99
-    config.max_episode_length = 200
-    config.target_network_mix = 0.001
-    config.exploration_steps = 100
-    config.noise_decay_interval = 10000
-    config.random_process_fn = \
-        lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
-    config.test_interval = 0
-    config.test_repetitions = 10
-    config.save_interval = 50
-    config.logger = Logger('./log', gym.logger)
-    run_episodes(DDPGAgent(config))
-
-def ddpg_lunar_lander():
-    task_fn = lambda: ContinuousLunarLander()
-    task = task_fn()
-    config = Config()
-    config.task_fn = task_fn
-    config.actor_network_fn = lambda: DeterministicActorNet(
-        task.state_dim, task.action_dim, F.tanh, 1, batch_norm=True)
-    config.critic_network_fn = lambda: DeterministicCriticNet(
-        task.state_dim, task.action_dim, batch_norm=True)
-    config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
-    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
-    config.critic_optimizer_fn =\
-        lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
-    config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
-    config.discount = 0.99
-    config.max_episode_length = 1000
-    config.target_network_mix = 0.001
-    config.exploration_steps = 100
-    config.noise_decay_interval = 10000
-    config.random_process_fn = \
-        lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
-    config.test_interval = 0
-    config.test_repetitions = 10
-    config.save_interval = 50
-    config.logger = Logger('./log', gym.logger)
-    run_episodes(DDPGAgent(config))
-
-def ddpg_walker():
-    task_fn = lambda: BipedalWalker()
-    task = task_fn()
-    config = Config()
-    config.task_fn = task_fn
-    config.actor_network_fn = lambda: DeterministicActorNet(
-        task.state_dim, task.action_dim, F.tanh, 1, gpu=True, batch_norm=False, non_linear=F.tanh)
-    config.critic_network_fn = lambda: DeterministicCriticNet(
-        task.state_dim, task.action_dim, gpu=True, batch_norm=False, non_linear=F.tanh)
-    config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
-    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
-    config.critic_optimizer_fn =\
-        lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
-    config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
-    config.discount = 0.99
-    config.min_epsilon = 0.1
-    config.max_episode_length = 999
-    config.target_network_mix = 0.001
-    config.exploration_steps = 10000
-    config.noise_decay_interval = 1000000
-    config.random_process_fn = \
-        lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
-    config.test_interval = 0
-    config.test_repetitions = 5
-    config.save_interval = 50
-    config.logger = Logger('./log', gym.logger)
-    run_episodes(DDPGAgent(config))
-
 def dqn_fruit():
    config = Config()
    config.task_fn = lambda: Fruit()
@@ -350,11 +199,38 @@ def hrmsdqn_fruit():
    config.episode_limit = 5000
    run_episodes(MSDQNAgent(config))

-def ppo_pendulum():
+def a3c_continuous():
    config = Config()
-    config.task_fn = lambda: Pendulum()
+    # config.task_fn = lambda: Pendulum()
+    config.task_fn = lambda: BipedalWalkerHardcore()
    task = config.task_fn()
-    config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim, gpu=False)
+    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001)
+    config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
+    config.network_fn = lambda: DisjointActorCriticNet(
+        # lambda: GaussianActorNet(task.state_dim, task.action_dim, unit_std=False, action_gate=F.tanh, action_scale=2.0),
+        lambda: GaussianActorNet(task.state_dim, task.action_dim, unit_std=True),
+        lambda: GaussianCriticNet(task.state_dim))
+    config.policy_fn = lambda: GaussianPolicy()
+    config.worker = ContinuousAdvantageActorCritic
+    config.discount = 0.99
+    config.max_episode_length = task.default_max_episode
+    config.num_workers = 8
+    config.update_interval = 20
+    config.test_interval = 1
+    config.test_repetitions = 1
+    config.entropy_weight = 0
+    config.gradient_clip = 40
+    config.logger = Logger('./log', gym.logger)
+    agent = AsyncAgent(config)
+    agent.run()
+
+def dppo_continuous():
+    config = Config()
+    # config.task_fn = lambda: Pendulum()
+    config.task_fn = lambda: BipedalWalkerHardcore()
+    task = config.task_fn()
+    config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim,
+                                                       gpu=False, unit_std=True)
    config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim, gpu=False)
    config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
@@ -368,7 +244,7 @@ def ppo_pendulum():
    config.num_workers = 8
    config.test_interval = 1
    config.test_repetitions = 1
-    config.max_episode_length = 200
+    config.max_episode_length = task.default_max_episode
    config.entropy_weight = 0
    config.gradient_clip = 40
    config.rollout_length = 10000
@@ -378,61 +254,32 @@ def ppo_pendulum():
    agent = AsyncAgent(config)
    agent.run()

-def ppo_lunar_lander():
+def ddpg_continuous():
+    task_fn = lambda: Pendulum()
+    task = task_fn()
    config = Config()
-    config.task_fn = lambda: ContinuousLunarLander()
-    task = config.task_fn()
-    config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim, gpu=False)
-    config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim, gpu=False)
+    config.task_fn = task_fn
+    config.actor_network_fn = lambda: DeterministicActorNet(
+        task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.relu, batch_norm=False)
+    config.critic_network_fn = lambda: DeterministicCriticNet(
+        task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False)
    config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
-    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
-    config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
-
-    config.policy_fn = lambda: GaussianPolicy()
-    config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=2048)
-    config.worker = ProximalPolicyOptimization
+    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
+    config.critic_optimizer_fn =\
+        lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
+    config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
    config.discount = 0.99
-    config.gae_tau = 0.97
-    config.num_workers = 8
-    config.test_interval = 1
-    config.test_repetitions = 1
-    config.max_episode_length = 1000
-    config.entropy_weight = 0
-    config.gradient_clip = 40
-    config.rollout_length = 10000
-    config.optimize_epochs = 1
-    config.ppo_ratio_clip = 0.2
+    config.max_episode_length = task.default_max_episode
+    config.target_network_mix = 0.001
+    config.exploration_steps = 100
+    config.noise_decay_interval = 10000
+    config.random_process_fn = \
+        lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
+    config.test_interval = 0
+    config.test_repetitions = 10
+    config.save_interval = 50
    config.logger = Logger('./log', gym.logger)
-    agent = AsyncAgent(config)
-    agent.run()
-
-def ppo_walker():
-    config = Config()
-    config.task_fn = lambda: BipedalWalker()
-    task = config.task_fn()
-    config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim, gpu=False)
-    config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim, gpu=False)
-    config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
-    config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
-    config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
-
-    config.policy_fn = lambda: GaussianPolicy()
-    config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=2048)
-    config.worker = ProximalPolicyOptimization
-    config.discount = 0.99
-    config.gae_tau = 0.97
-    config.num_workers = 8
-    config.test_interval = 1
-    config.test_repetitions = 1
-    config.max_episode_length = 999
-    config.entropy_weight = 0
-    config.gradient_clip = 40
-    config.rollout_length = 10000
-    config.optimize_epochs = 1
-    config.ppo_ratio_clip = 0.2
-    config.logger = Logger('./log', gym.logger)
-    agent = AsyncAgent(config)
-    agent.run()
+    run_episodes(DDPGAgent(config))

 if __name__ == '__main__':
    # gym.logger.setLevel(logging.DEBUG)
@@ -441,15 +288,9 @@ if __name__ == '__main__':
    # dqn_cart_pole()
    # async_cart_pole()
    # a3c_cart_pole()
-    # a3c_pendulum()
-    # a3c_lunar_lander()
-    # a3c_walker()
-    # ddpg_pendulum()
-    # ddpg_lunar_lander()
-    # ddpg_walker()
-    # ppo_pendulum()
-    # ppo_lunar_lander()
-    ppo_walker()
+    # a3c_continuous()
+    # dppo_continuous()
+    ddpg_continuous()

    # dqn_fruit()
    # hrdqn_fruit()
@@ -145,7 +145,7 @@ class GaussianActorNet(nn.Module, BasicNet):
            log_std = self.action_log_std.expand_as(mean)
            std = log_std.exp()
        else:
-            std = F.softplus(self.fc_std(x) + 1e-5)
+            std = F.softplus(self.action_std(phi) + 1e-5)
            log_std = std.log()
        return mean, std, log_std