diff --git a/python/ray/tune/ray_trial_executor.py b/python/ray/tune/ray_trial_executor.py index 85fa86afd..b2d97cee6 100644 --- a/python/ray/tune/ray_trial_executor.py +++ b/python/ray/tune/ray_trial_executor.py @@ -9,6 +9,7 @@ import time import traceback import ray +from ray.tune.error import TuneError from ray.tune.logger import NoopLogger from ray.tune.trial import Trial, Resources, Checkpoint from ray.tune.trial_executor import TrialExecutor @@ -270,6 +271,11 @@ class RayTrialExecutor(TrialExecutor): logger.warning("Cluster resources not detected. Retrying...") time.sleep(0.5) + if not resources or "CPU" not in resources: + raise TuneError("Cluster resources cannot be detected. " + "You can resume this experiment by passing in " + "`resume=True` to `run_experiments`.") + resources = resources.copy() num_cpus = resources.pop("CPU") num_gpus = resources.pop("GPU")