[tune] update params for optimizer in reset_config (#6522)

* reset config update lr

* add default

* Update pbt_dcgan_mnist.py

* Update pbt_convnet_example.py

Co-authored-by: Richard Liaw <rliaw@berkeley.edu>

python/ray/tune/examples/pbt_convnet_example.py

@@ -52,11 +52,13 @@ class PytorchTrainble(tune.Trainable):
         self.model.load_state_dict(torch.load(checkpoint_path))
 
     def reset_config(self, new_config):
-        del self.optimizer
-        self.optimizer = optim.SGD(
-            self.model.parameters(),
-            lr=new_config.get("lr", 0.01),
-            momentum=new_config.get("momentum", 0.9))
+        for param_group in self.optimizer.param_groups:
+            if "lr" in new_config:
+                param_group["lr"] = new_config["lr"]
+            if "momentum" in new_config:
+                param_group["momentum"] = new_config["momentum"]
+
+        self.config = new_config
         return True