[tune] Retry failed tasks with checkpointing disabled (#6126)

* Allow recovery for failed tasks without checkpointing

* Update docs

python/ray/tune/trial.py

@@ -304,15 +304,14 @@ class Trial(object):
         self._checkpoint.value = None
 
     def should_recover(self):
-        """Returns whether the trial qualifies for restoring.
+        """Returns whether the trial qualifies for retrying.
 
-        This is if a checkpoint frequency is set and has not failed more than
-        max_failures. This may return true even when there may not yet
-        be a checkpoint.
+        This is if the trial has not failed more than max_failures. Note this
+        may return true even when there is no checkpoint, either because
+        `self.checkpoint_freq` is `0` or because the trial failed before
+        a checkpoint has been made.
         """
-        return (self.checkpoint_freq > 0
-                and (self.num_failures < self.max_failures
-                     or self.max_failures < 0))
+        return self.num_failures < self.max_failures or self.max_failures < 0
 
     def update_last_result(self, result, terminate=False):
         result.update(trial_id=self.trial_id, done=terminate)

python/ray/tune/trial_runner.py

@@ -535,8 +535,7 @@ class TrialRunner(object):
                 stop_logger=False)
             trial.result_logger.flush()
             if self.trial_executor.has_resources(trial.resources):
-                logger.info("Attempting to recover"
-                            " trial state from last checkpoint.")
+                logger.info("Attempting to recover trial.")
                 self.trial_executor.start_trial(trial)
                 if trial.status == Trial.ERROR:
                     raise RuntimeError("Trial did not start correctly.")

python/ray/tune/tune.py

@@ -150,10 +150,10 @@ def run(run_or_experiment,
             for individual trials.
         export_formats (list): List of formats that exported at the end of
             the experiment. Default is None.
-        max_failures (int): Try to recover a trial from its last
-            checkpoint at least this many times. Only applies if
-            checkpointing is enabled. Setting to -1 will lead to infinite
-            recovery retries. Defaults to 3.
+        max_failures (int): Try to recover a trial at least this many times.
+            Ray will recover from the latest checkpoint if present.
+            Setting to -1 will lead to infinite recovery retries.
+            Setting to 0 will disable retries. Defaults to 3.
         restore (str): Path to checkpoint. Only makes sense to set if
             running 1 trial. Defaults to None.
         search_alg (SearchAlgorithm): Search Algorithm. Defaults to