[tune] Cross-Node Recovery (#3725)

Augments trial restore to also check if the runner is at the same
location. If not, the checkpoint files are pushed onto the new location.

python/ray/tune/trial.py

@@ -196,8 +196,8 @@ class Trial(object):
         self._checkpoint = Checkpoint(
             storage=Checkpoint.DISK, value=restore_path)
         self.status = Trial.PENDING
-        self.location = None
         self.logdir = None
+        self.runner = None
         self.result_logger = None
         self.last_debug = 0
         self.trial_id = Trial.generate_id() if trial_id is None else trial_id
@@ -241,6 +241,14 @@ class Trial(object):
                 custom_loggers=self.custom_loggers,
                 sync_function=self.sync_function)
 
+    def sync_logger_to_new_location(self, worker_ip):
+        """Updates the logger location.
+
+        Also pushes logdir to worker_ip, allowing for cross-node recovery.
+        """
+        if self.result_logger:
+            self.result_logger.sync_results_to_new_location(worker_ip)
+
     def close_logger(self):
         """Close logger."""