wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-11 11:24:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] Component notification on node failure + Tests (#3414)

Browse Source 
Changes include:
 - Notify Components on Requeue
 - Slight refactoring of Node Failure handling
 - Better tests
This commit is contained in:
Richard Liaw
2018-12-04 14:47:31 -08:00
committed by GitHub
parent ce355d13d4
commit 9d0bd50e78
8 changed files with 180 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/ray_trial_executor.py
+13 -4
View File
@@ -110,19 +110,27 @@ class RayTrialExecutor(TrialExecutor):
if stop_logger:
trial.close_logger()
def start_trial(self, trial, checkpoint_obj=None):
"""Starts the trial."""
def start_trial(self, trial, checkpoint=None):
"""Starts the trial.
Will not return resources if trial repeatedly fails on start.
Args:
trial (Trial): Trial to be started.
checkpoint (Checkpoint): A Python object or path storing the state
of trial.
"""
self._commit_resources(trial.resources)
try:
self._start_trial(trial, checkpoint_obj)
self._start_trial(trial, checkpoint)
except Exception:
logger.exception("Error stopping runner - retrying...")
error_msg = traceback.format_exc()
time.sleep(2)
self._stop_trial(trial, error=True, error_msg=error_msg)
try:
self._start_trial(trial)
self._start_trial(trial, checkpoint)
except Exception:
logger.exception("Error starting runner, aborting!")
error_msg = traceback.format_exc()
@@ -140,6 +148,7 @@ class RayTrialExecutor(TrialExecutor):
self._stop_trial(
trial, error=error, error_msg=error_msg, stop_logger=stop_logger)
if prior_status == Trial.RUNNING:
logger.debug("Returning resources for this trial.")
self._return_resources(trial.resources)
out = self._find_item(self._running, trial)
for result_id in out: