mirror of
https://github.com/wassname/ray.git
synced 2026-07-27 11:26:41 +08:00
[autoscaler] Add kill and get IP commands to CLI for testing (#3731)
## What do these changes do? Adds 2 commands to the CLI that take in an autoscaler config: 1. Kill a random ray node in the cluster. 2. Get all the worker node IP addresses. These commands are both for testing and are not recommended for normal use. ## Related issue number Closes #3685.
This commit is contained in:
committed by
Richard Liaw
parent
574f0b73bc
commit
cc5ecd71c5
@@ -11,6 +11,7 @@ import time
|
||||
import sys
|
||||
import click
|
||||
import logging
|
||||
import random
|
||||
|
||||
import yaml
|
||||
try: # py3
|
||||
@@ -94,6 +95,35 @@ def teardown_cluster(config_file, yes, workers_only, override_cluster_name):
|
||||
nodes = provider.nodes({TAG_RAY_NODE_TYPE: "worker"})
|
||||
|
||||
|
||||
def kill_node(config_file, yes, override_cluster_name):
|
||||
"""Kills a random Raylet worker."""
|
||||
|
||||
config = yaml.load(open(config_file).read())
|
||||
if override_cluster_name is not None:
|
||||
config["cluster_name"] = override_cluster_name
|
||||
config = _bootstrap_config(config)
|
||||
|
||||
confirm("This will kill a node in your cluster", yes)
|
||||
|
||||
provider = get_node_provider(config["provider"], config["cluster_name"])
|
||||
nodes = provider.nodes({TAG_RAY_NODE_TYPE: "worker"})
|
||||
node = random.choice(nodes)
|
||||
logger.info("Terminating worker {}".format(node))
|
||||
updater = NodeUpdaterProcess(
|
||||
node,
|
||||
config["provider"],
|
||||
config["auth"],
|
||||
config["cluster_name"],
|
||||
config["file_mounts"], [],
|
||||
"",
|
||||
redirect_output=False)
|
||||
|
||||
_exec(updater, "ray stop", False, False)
|
||||
|
||||
time.sleep(5)
|
||||
return provider.external_ip(node)
|
||||
|
||||
|
||||
def get_or_create_head_node(config, config_file, no_restart, restart_only, yes,
|
||||
override_cluster_name):
|
||||
"""Create the cluster head node, which in turn creates the workers."""
|
||||
@@ -343,6 +373,17 @@ def get_head_node_ip(config_file, override_cluster_name):
|
||||
return provider.external_ip(head_node)
|
||||
|
||||
|
||||
def get_worker_node_ips(config_file, override_cluster_name):
|
||||
"""Returns worker node IPs for given configuration file."""
|
||||
|
||||
config = yaml.load(open(config_file).read())
|
||||
if override_cluster_name is not None:
|
||||
config["cluster_name"] = override_cluster_name
|
||||
provider = get_node_provider(config["provider"], config["cluster_name"])
|
||||
nodes = provider.nodes({TAG_RAY_NODE_TYPE: "worker"})
|
||||
return [provider.external_ip(node) for node in nodes]
|
||||
|
||||
|
||||
def _get_head_node(config,
|
||||
config_file,
|
||||
override_cluster_name,
|
||||
|
||||
Reference in New Issue
Block a user