[RLlib] DDPG PyTorch version. (#7953)

The DDPG/TD3 algorithms currently do not have a PyTorch implementation. This PR adds PyTorch support for DDPG/TD3 to RLlib.
This PR:
- Depends on the re-factor PR for DDPG (Functional Algorithm API).
- Adds learning regression tests for the PyTorch version of DDPG and a DDPG (torch)
- Updates the documentation to reflect that DDPG and TD3 now support PyTorch.

* Learning Pendulum-v0 on torch version (same config as tf). Wall time a little slower (~20% than tf).
* Fix GPU target model problem.

rllib/utils/exploration/ornstein_uhlenbeck_noise.py

@@ -153,10 +153,10 @@ class OrnsteinUhlenbeckNoise(GaussianNoise):
                 ou_new = self.ou_theta * -self.ou_state + \
                     self.ou_sigma * gaussian_sample
                 self.ou_state += ou_new
-                high_low = torch.from_numpy(self.action_space.high -
-                                            self.action_space.low).to(
-                                                self.device)
-                noise = scale * self.ou_base_scale * self.ou_state * high_low
+                high_m_low = torch.from_numpy(
+                    self.action_space.high - self.action_space.low). \
+                    to(self.device)
+                noise = scale * self.ou_base_scale * self.ou_state * high_m_low
                 action = torch.clamp(det_actions + noise,
                                      self.action_space.low[0],
                                      self.action_space.high[0])

rllib/utils/exploration/tests/test_explorations.py

@@ -24,15 +24,14 @@ def do_test_explorations(run,
                          expected_mean_action=None):
     """Calls an Agent's `compute_actions` with different `explore` options."""
 
-    config = config.copy()
+    core_config = config.copy()
     if run not in [a3c.A3CTrainer]:
-        config["num_workers"] = 0
+        core_config["num_workers"] = 0
 
     # Test all frameworks.
-    for fw in framework_iterator(config):
+    for fw in framework_iterator(core_config):
         if fw == "torch" and \
-                run in [ddpg.DDPGTrainer, impala.ImpalaTrainer,
-                        sac.SACTrainer, td3.TD3Trainer]:
+                run in [impala.ImpalaTrainer, sac.SACTrainer]:
             continue
         elif fw == "eager" and run in [
                 ddpg.DDPGTrainer, sac.SACTrainer, td3.TD3Trainer
@@ -44,14 +43,15 @@ def do_test_explorations(run,
         # Test for both the default Agent's exploration AND the `Random`
         # exploration class.
         for exploration in [None, "Random"]:
+            local_config = core_config.copy()
             if exploration == "Random":
                 # TODO(sven): Random doesn't work for IMPALA yet.
                 if run is impala.ImpalaTrainer:
                     continue
-                config["exploration_config"] = {"type": "Random"}
+                local_config["exploration_config"] = {"type": "Random"}
             print("exploration={}".format(exploration or "default"))
 
-            trainer = run(config=config, env=env)
+            trainer = run(config=local_config, env=env)
 
             # Make sure all actions drawn are the same, given same
             # observations.

rllib/utils/exploration/tests/test_parameter_noise.py

@@ -40,7 +40,7 @@ class TestParameterNoise(unittest.TestCase):
 
             config = core_config.copy()
 
-            # DQN with ParameterNoise exploration (config["explore"]=True).
+            # Algo with ParameterNoise exploration (config["explore"]=True).
             # ----
             config["exploration_config"] = {"type": "ParameterNoise"}
             config["explore"] = True

rllib/utils/framework.py

@@ -192,7 +192,9 @@ def get_variable(value,
             tf_name, initializer=value, dtype=dtype, trainable=trainable)
     elif framework == "torch" and torch_tensor is True:
         torch, _ = try_import_torch()
-        var_ = torch.from_numpy(value).to(device)
+        var_ = torch.from_numpy(value)
+        if device:
+            var_ = var_.to(device)
         var_.requires_grad = trainable
         return var_
     # torch or None: Return python primitive.

rllib/utils/torch_ops.py

@@ -21,6 +21,14 @@ def huber_loss(x, delta=1.0):
         torch.pow(x, 2.0) * 0.5, delta * (torch.abs(x) - 0.5 * delta))
 
 
+def l2_loss(x):
+    """Computes half the L2 norm of a tensor without the sqrt.
+
+    output = sum(x ** 2) / 2
+    """
+    return torch.sum(torch.pow(x, 2.0)) / 2.0
+
+
 def reduce_mean_ignore_inf(x, axis):
     """Same as torch.mean() but ignores -inf values."""
     mask = torch.ne(x, float("-inf"))
@@ -28,17 +36,16 @@ def reduce_mean_ignore_inf(x, axis):
     return torch.sum(x_zeroed, axis) / torch.sum(mask.float(), axis)
 
 
-def minimize_and_clip(optimizer, objective, var_list, clip_val=10):
-    """Minimized `objective` using `optimizer` w.r.t. variables in
-    `var_list` while ensure the norm of the gradients for each
-    variable is clipped to `clip_val`
+def minimize_and_clip(optimizer, clip_val=10):
+    """Clips gradients found in `optimizer.param_groups` to given value.
+
+    Ensures the norm of the gradients for each variable is clipped to
+    `clip_val`
     """
-    gradients = optimizer.compute_gradients(objective, var_list=var_list)
-    for i, (grad, var) in enumerate(gradients):
-        if grad is not None:
-            gradients[i] = (torch.nn.utils.clip_grad_norm_(grad, clip_val),
-                            var)
-    return gradients
+    for param_group in optimizer.param_groups:
+        for p in param_group["params"]:
+            if p.grad is not None:
+                torch.nn.utils.clip_grad_norm_(p.grad, clip_val)
 
 
 def sequence_mask(lengths, maxlen, dtype=None):