Benchmark DDPG with Roboschool

README.md

@@ -49,10 +49,10 @@ variance unbounded, which is also included in the implementation.
 
 ## DDPG
 
-![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DDPG-Pendulum-v0.png)
+![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/DDPG.png)
 
 Extra caution is necessary when computing gradients, the [repo](https://github.com/ghliu/pytorch-ddpg) I referred
-seems to have critical bugs. Anyway DDPG is fairly unstable.
+seems to have critical bugs. DDPG is not very stable. 
 
 ## DPPO
 
@@ -69,9 +69,12 @@ I use 8 threads and a two tanh hidden layer network, each hidden layer has 64 hi
 
 # Dependency
 * Open AI gym
-* PyTorch v0.12 + Python 2.7 (Well tested)
-* PyTorch v0.2 + Python 3.6 (Experimental)
+* [RoboSchool](https://github.com/openai/roboschool) (Optional)
+* PyTorch v0.12 
+* Python 2.7 or Python 3.6
 * Tensorflow (Optional, but tensorboard is awesome)
+> If you want to use Roboschool, you have to use python3. And don't try to use Roboschool with parallelized algorithms,
+> there is a known [critical bug](https://github.com/openai/roboschool/issues/86).
 
 # Usage
 Detailed usage and all training parameters can be found in ```main.py```.

agent/DDPG_agent.py

@@ -61,7 +61,7 @@ class DDPGAgent:
             done = (done or (config.max_episode_length and steps >= config.max_episode_length))
             next_state = self.state_normalizer(next_state)
             total_reward += reward
-            reward = np.asscalar(self.reward_normalizer(np.array([reward])))
+            # reward = self.reward_normalizer(reward)
 
             if not deterministic:
                 self.replay.feed([state, action, reward, next_state, int(done)])

component/task.py

@@ -7,10 +7,6 @@ import gym
 import sys
 import numpy as np
 from .atari_wrapper import *
-try:
-    import roboschool
-except:
-    gym.logger.info('Roboschool not found')
 
 class BasicTask:
     def __init__(self):
@@ -168,6 +164,7 @@ class ContinuousLunarLander(BasicTask):
 
 class Roboschool(BasicTask):
     def __init__(self, name, success_threshold=sys.maxsize, max_episode_steps=None):
+        import roboschool
         BasicTask.__init__(self)
         self.name = name
         self.env = gym.make(self.name)

main.py

@@ -259,6 +259,7 @@ def ddpg_continuous():
     config = Config()
     # config.task_fn = lambda: Pendulum()
     config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1')
+    # config.task_fn = lambda: Roboschool('RoboschoolReacher-v1')
     task = config.task_fn()
     config.actor_network_fn = lambda: DeterministicActorNet(
         task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.relu, batch_norm=False)
@@ -274,6 +275,7 @@ def ddpg_continuous():
     config.target_network_mix = 0.001
     config.exploration_steps = 100
     config.noise_decay_interval = 10000
+    config.min_epsilon = 0.1
     config.random_process_fn = \
         lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
     config.test_interval = 0
@@ -290,8 +292,8 @@ if __name__ == '__main__':
     # async_cart_pole()
     # a3c_cart_pole()
     # a3c_continuous()
-    dppo_continuous()
-    # ddpg_continuous()
+    # dppo_continuous()
+    ddpg_continuous()
 
     # dqn_fruit()
     # hrdqn_fruit()

network/continuous_action_network.py

@@ -30,7 +30,7 @@ class DeterministicActorNet(nn.Module, BasicNet):
 
         self.batch_norm = batch_norm
         BasicNet.__init__(self, None, gpu, False)
-        # self.init_weights()
+        self.init_weights()
 
     def init_weights(self):
         bound = 3e-3
@@ -84,7 +84,7 @@ class DeterministicCriticNet(nn.Module, BasicNet):
         self.batch_norm = batch_norm
 
         BasicNet.__init__(self, None, gpu, False)
-        # self.init_weights()
+        self.init_weights()
 
     def init_weights(self):
         bound = 3e-3

utils/normalizer.py

@@ -11,11 +11,19 @@ class Normalizer:
         self.stats = SharedStats(o_size)
 
     def __call__(self, o_):
-        o = torch.FloatTensor(o_)
+        if np.isscalar(o_):
+            o = torch.FloatTensor([o_])
+        else:
+            o = torch.FloatTensor(o_)
         self.stats.feed(o)
         std = (self.stats.v + 1e-6) ** .5
         o = (o - self.stats.m) / std
-        return o.numpy().reshape(o_.shape)
+        o = o.numpy()
+        if np.isscalar(o_):
+            o = np.asscalar(o)
+        else:
+            o = o.reshape(o_.shape)
+        return o
 
 class StaticNormalizer:
     def __init__(self, o_size):