wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-06 05:16:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] Change log handling for Tune (#3661)

Browse Source 
Also provides a small retry mechanism for a transient error as reported
by #3340.

Closes #3653.
This commit is contained in:
Richard Liaw
2019-01-06 13:20:10 -08:00
committed by GitHub
parent 681e8cd3fd
commit 8934e37a78
5 changed files with 33 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/ray_trial_executor.py
+8 -2
View File
@@ -234,10 +234,16 @@ class RayTrialExecutor(TrialExecutor):
assert self._committed_resources.cpu >= 0
assert self._committed_resources.gpu >= 0
def _update_avail_resources(self):
resources = ray.global_state.cluster_resources()
def _update_avail_resources(self, num_retries=5):
for i in range(num_retries):
resources = ray.global_state.cluster_resources()
if not resources:
logger.warning("Cluster resources not detected. Retrying...")
time.sleep(0.5)
num_cpus = resources["CPU"]
num_gpus = resources["GPU"]
self._avail_resources = Resources(int(num_cpus), int(num_gpus))
self._resources_initialized = True