[tune] Cross-Node Recovery (#3725)

Augments trial restore to also check if the runner is at the same
location. If not, the checkpoint files are pushed onto the new location.

python/ray/tune/trainable.py

@@ -104,6 +104,10 @@ class Trainable(object):
 
         return ""
 
+    def current_ip(self):
+        self._local_ip = ray.services.get_node_ip_address()
+        return self._local_ip
+
     def train(self):
         """Runs one logical iteration of training.