[tune] Stop-gap fix for PBT checkpointing (#7794)

* Fix PBT

* lint

* reset

* rm

* tests

Co-authored-by: Richard Liaw <rliaw@berkeley.edu>

python/ray/tune/durable_trainable.py

@@ -1,8 +1,11 @@
+import logging
 import os
 
 from ray.tune.trainable import Trainable, TrainableUtil
 from ray.tune.syncer import get_cloud_sync_client
 
+logger = logging.getLogger(__name__)
+
 
 class DurableTrainable(Trainable):
     """Abstract class for a remote-storage backed fault-tolerant Trainable.
@@ -57,7 +60,6 @@ class DurableTrainable(Trainable):
             if checkpoint_dir.starts_with(os.path.abspath(self.logdir)):
                 raise ValueError("`checkpoint_dir` must be `self.logdir`, or "
                                  "a sub-directory.")
-
         checkpoint_path = super(DurableTrainable, self).save(checkpoint_dir)
         self.storage_client.sync_up(self.logdir, self.remote_checkpoint_dir)
         self.storage_client.wait()
@@ -81,9 +83,15 @@ class DurableTrainable(Trainable):
         Args:
             checkpoint_path (str): Local path to checkpoint.
         """
+        try:
+            local_dirpath = TrainableUtil.find_checkpoint_dir(checkpoint_path)
+        except FileNotFoundError:
+            logger.warning(
+                "Trial %s: checkpoint path not found during "
+                "garbage collection. See issue #6697.", self.trial_id)
+        else:
+            self.storage_client.delete(self._storage_path(local_dirpath))
         super(DurableTrainable, self).delete_checkpoint(checkpoint_path)
-        local_dirpath = TrainableUtil.find_checkpoint_dir(checkpoint_path)
-        self.storage_client.delete(self._storage_path(local_dirpath))
 
     def _create_storage_client(self):
         """Returns a storage client."""