From 900a48c19c443e51289e66a19264a4bc55a46678 Mon Sep 17 00:00:00 2001 From: Amog Kamsetty Date: Fri, 6 Nov 2020 15:01:55 -0800 Subject: [PATCH] [Tune] Better warnings/exceptions for fail_fast='raise' (#11842) --- python/ray/tune/trial_runner.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/ray/tune/trial_runner.py b/python/ray/tune/trial_runner.py index 400f42d8f..09f34f82e 100644 --- a/python/ray/tune/trial_runner.py +++ b/python/ray/tune/trial_runner.py @@ -6,6 +6,7 @@ import os import time import traceback import types +import warnings import ray.cloudpickle as cloudpickle from ray.services import get_node_ip_address @@ -157,7 +158,7 @@ class TrialRunner: if isinstance(self._fail_fast, str): self._fail_fast = self._fail_fast.upper() if self._fail_fast == TrialRunner.RAISE: - logger.warning( + warnings.warn( "fail_fast='raise' detected. Be careful when using this " "mode as resources (such as Ray processes, " "file descriptors, and temporary files) may not be " @@ -474,7 +475,7 @@ class TrialRunner: wait_for_trial = trials_done and not self._search_alg.is_finished() # Only fetch a new trial if we have no pending trial if not any(trial.status == Trial.PENDING for trial in self._trials) \ - or wait_for_trial: + or wait_for_trial: self._update_trial_queue(blocking=wait_for_trial) with warn_if_slow("choose_trial_to_run"): trial = self._scheduler_alg.choose_trial_to_run(self) @@ -625,9 +626,12 @@ class TrialRunner: else: self._execute_action(trial, decision) except Exception: - logger.exception("Trial %s: Error processing event.", trial) + error_msg = "Trial %s: Error processing event." % trial if self._fail_fast == TrialRunner.RAISE: + logger.error(error_msg) raise + else: + logger.exception(error_msg) self._process_trial_failure(trial, traceback.format_exc()) def _validate_result_metrics(self, result):