diff --git a/doc/source/tune-usage.rst b/doc/source/tune-usage.rst index bc9589ec0..c5d2f42ec 100644 --- a/doc/source/tune-usage.rst +++ b/doc/source/tune-usage.rst @@ -412,9 +412,9 @@ The checkpoint will be saved at a path that looks like ``local_dir/exp_name/tria Fault Tolerance --------------- -Tune will automatically restart trials from the last checkpoint in case of trial failures/error (if ``max_failures`` is set), both in the single node and distributed setting. +Tune will automatically restart trials in case of trial failures/error (if ``max_failures != 0``), both in the single node and distributed setting. -In the distributed setting, if using the autoscaler with ``rsync`` enabled, Tune will automatically sync the trial folder with the driver. For example, if a node is lost while a trial (specifically, the corresponding Trainable actor of the trial) is still executing on that node and a checkpoint of the trial exists, Tune will wait until available resources are available to begin executing the trial again. +Tune will restore trials from the latest checkpoint, where available. In the distributed setting, if using the autoscaler with ``rsync`` enabled, Tune will automatically sync the trial folder with the driver. For example, if a node is lost while a trial (specifically, the corresponding Trainable actor of the trial) is still executing on that node and a checkpoint of the trial exists, Tune will wait until available resources are available to begin executing the trial again. If the trial/actor is placed on a different node, Tune will automatically push the previous checkpoint file to that node and restore the remote trial actor state, allowing the trial to resume from the latest checkpoint even after failure. diff --git a/python/ray/tune/trial.py b/python/ray/tune/trial.py index 840533c7e..5568a659a 100644 --- a/python/ray/tune/trial.py +++ b/python/ray/tune/trial.py @@ -304,15 +304,14 @@ class Trial(object): self._checkpoint.value = None def should_recover(self): - """Returns whether the trial qualifies for restoring. + """Returns whether the trial qualifies for retrying. - This is if a checkpoint frequency is set and has not failed more than - max_failures. This may return true even when there may not yet - be a checkpoint. + This is if the trial has not failed more than max_failures. Note this + may return true even when there is no checkpoint, either because + `self.checkpoint_freq` is `0` or because the trial failed before + a checkpoint has been made. """ - return (self.checkpoint_freq > 0 - and (self.num_failures < self.max_failures - or self.max_failures < 0)) + return self.num_failures < self.max_failures or self.max_failures < 0 def update_last_result(self, result, terminate=False): result.update(trial_id=self.trial_id, done=terminate) diff --git a/python/ray/tune/trial_runner.py b/python/ray/tune/trial_runner.py index 6814888e5..f8c7feac5 100644 --- a/python/ray/tune/trial_runner.py +++ b/python/ray/tune/trial_runner.py @@ -535,8 +535,7 @@ class TrialRunner(object): stop_logger=False) trial.result_logger.flush() if self.trial_executor.has_resources(trial.resources): - logger.info("Attempting to recover" - " trial state from last checkpoint.") + logger.info("Attempting to recover trial.") self.trial_executor.start_trial(trial) if trial.status == Trial.ERROR: raise RuntimeError("Trial did not start correctly.") diff --git a/python/ray/tune/tune.py b/python/ray/tune/tune.py index 69acbfba8..efd8dd715 100644 --- a/python/ray/tune/tune.py +++ b/python/ray/tune/tune.py @@ -150,10 +150,10 @@ def run(run_or_experiment, for individual trials. export_formats (list): List of formats that exported at the end of the experiment. Default is None. - max_failures (int): Try to recover a trial from its last - checkpoint at least this many times. Only applies if - checkpointing is enabled. Setting to -1 will lead to infinite - recovery retries. Defaults to 3. + max_failures (int): Try to recover a trial at least this many times. + Ray will recover from the latest checkpoint if present. + Setting to -1 will lead to infinite recovery retries. + Setting to 0 will disable retries. Defaults to 3. restore (str): Path to checkpoint. Only makes sense to set if running 1 trial. Defaults to None. search_alg (SearchAlgorithm): Search Algorithm. Defaults to