wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-28 16:46:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix "failed to create head node" issue (#6304)

Browse Source 
* Fix failed to create head node issue

* comments
This commit is contained in:
Edward Oakes
2019-12-02 15:22:00 -08:00
committed by GitHub
parent 2a3adf2d70
commit dff6017272
1 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/autoscaler/commands.py
+10 -3
View File
@@ -199,9 +199,16 @@ def get_or_create_head_node(config, config_file, no_restart, restart_only, yes,
config["cluster_name"])
provider.create_node(config["head_node"], head_node_tags, 1)
nodes = provider.non_terminated_nodes(head_node_tags)
assert len(nodes) == 1, "Failed to create head node."
head_node = nodes[0]
start = time.time()
head_node = None
while True:
if time.time() - start > 5:
raise RuntimeError("Failed to create head node.")
nodes = provider.non_terminated_nodes(head_node_tags)
if len(nodes) == 1:
head_node = nodes[0]
break
time.sleep(1)
# TODO(ekl) right now we always update the head node even if the hash
# matches. We could prompt the user for what they want to do here.