wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-01 12:51:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[autoscaler] Add hard kill and monitor commands (#5082)

Browse Source 
* Add hard kill and monitor commands

* better_commands

* Update python/ray/scripts/scripts.py

Co-Authored-By: Kristian Hartikainen <kristian.hartikainen@gmail.com>
This commit is contained in:
Richard Liaw
2019-07-06 21:52:55 -07:00
committed by Eric Liang
co-authored by Kristian Hartikainen
parent 445bcb29b0
commit 1798d4f077
2 changed files with 51 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/autoscaler/commands.py
+22 -13
View File
@@ -116,7 +116,7 @@ def teardown_cluster(config_file, yes, workers_only, override_cluster_name):
provider.cleanup()
def kill_node(config_file, yes, override_cluster_name):
def kill_node(config_file, yes, hard, override_cluster_name):
"""Kills a random Raylet worker."""
config = yaml.safe_load(open(config_file).read())
@@ -131,19 +131,21 @@ def kill_node(config_file, yes, override_cluster_name):
nodes = provider.non_terminated_nodes({TAG_RAY_NODE_TYPE: "worker"})
node = random.choice(nodes)
logger.info("kill_node: Terminating worker {}".format(node))
if hard:
provider.terminate_node(node)
else:
updater = NodeUpdaterThread(
node_id=node,
provider_config=config["provider"],
provider=provider,
auth_config=config["auth"],
cluster_name=config["cluster_name"],
file_mounts=config["file_mounts"],
initialization_commands=[],
setup_commands=[],
runtime_hash="")
updater = NodeUpdaterThread(
node_id=node,
provider_config=config["provider"],
provider=provider,
auth_config=config["auth"],
cluster_name=config["cluster_name"],
file_mounts=config["file_mounts"],
initialization_commands=[],
setup_commands=[],
runtime_hash="")
_exec(updater, "ray stop", False, False)
_exec(updater, "ray stop", False, False)
time.sleep(5)
@@ -157,6 +159,13 @@ def kill_node(config_file, yes, override_cluster_name):
return node_ip
def monitor_cluster(cluster_config_file, num_lines, override_cluster_name):
"""Kills a random Raylet worker."""
cmd = "tail -n {} -f /tmp/ray/session_*/logs/monitor*".format(num_lines)
exec_cluster(cluster_config_file, cmd, False, False, False, False, False,
override_cluster_name, None)
def get_or_create_head_node(config, config_file, no_restart, restart_only, yes,
override_cluster_name):
"""Create the cluster head node, which in turn creates the workers."""