[autoscaler] Add kill and get IP commands to CLI for testing (#3731)

## What do these changes do?

Adds 2 commands to the CLI that take in an autoscaler config:
1. Kill a random ray node in the cluster.
2. Get all the worker node IP addresses.

These commands are both for testing and are not recommended for normal use.

## Related issue number
Closes #3685.
This commit is contained in:
Stephanie Wang
2019-01-10 22:06:57 -08:00
committed by Richard Liaw
parent 574f0b73bc
commit cc5ecd71c5
2 changed files with 85 additions and 7 deletions
+41
View File
@@ -11,6 +11,7 @@ import time
import sys
import click
import logging
import random
import yaml
try: # py3
@@ -94,6 +95,35 @@ def teardown_cluster(config_file, yes, workers_only, override_cluster_name):
nodes = provider.nodes({TAG_RAY_NODE_TYPE: "worker"})
def kill_node(config_file, yes, override_cluster_name):
"""Kills a random Raylet worker."""
config = yaml.load(open(config_file).read())
if override_cluster_name is not None:
config["cluster_name"] = override_cluster_name
config = _bootstrap_config(config)
confirm("This will kill a node in your cluster", yes)
provider = get_node_provider(config["provider"], config["cluster_name"])
nodes = provider.nodes({TAG_RAY_NODE_TYPE: "worker"})
node = random.choice(nodes)
logger.info("Terminating worker {}".format(node))
updater = NodeUpdaterProcess(
node,
config["provider"],
config["auth"],
config["cluster_name"],
config["file_mounts"], [],
"",
redirect_output=False)
_exec(updater, "ray stop", False, False)
time.sleep(5)
return provider.external_ip(node)
def get_or_create_head_node(config, config_file, no_restart, restart_only, yes,
override_cluster_name):
"""Create the cluster head node, which in turn creates the workers."""
@@ -343,6 +373,17 @@ def get_head_node_ip(config_file, override_cluster_name):
return provider.external_ip(head_node)
def get_worker_node_ips(config_file, override_cluster_name):
"""Returns worker node IPs for given configuration file."""
config = yaml.load(open(config_file).read())
if override_cluster_name is not None:
config["cluster_name"] = override_cluster_name
provider = get_node_provider(config["provider"], config["cluster_name"])
nodes = provider.nodes({TAG_RAY_NODE_TYPE: "worker"})
return [provider.external_ip(node) for node in nodes]
def _get_head_node(config,
config_file,
override_cluster_name,