wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-15 12:45:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Auto-scale ray clusters based on GCS load metrics (#1348)

Browse Source 
This adds (experimental) auto-scaling support for Ray clusters based on GCS load metrics. The auto-scaling algorithm is as follows:

Based on current (instantaneous) load information, we compute the approximate number of "used workers". This is based on the bottleneck resource, e.g. if 8/8 GPUs are used in a 8-node cluster but all the CPUs are idle, the number of used nodes is still counted as 8. This number can also be fractional.
We scale that number by 1 / target_utilization_fraction and round up to determine the target cluster size (subject to the max_workers constraint). The autoscaler control loop takes care of launching new nodes until the target cluster size is met.
When a node is idle for more than idle_timeout_minutes, we remove it from the cluster if that would not drop the cluster size below min_workers.
Note that we'll need to update the wheel in the example yaml file after this PR is merged.
This commit is contained in:
Eric Liang
2017-12-31 14:39:57 -08:00
committed by GitHub
parent e970e24ea5
commit b6c42f96be
12 changed files with 657 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/autoscaler_test.py
+162 -19
View File
@@ -9,7 +9,7 @@ import unittest
import yaml
import ray
from ray.autoscaler.autoscaler import StandardAutoscaler
from ray.autoscaler.autoscaler import StandardAutoscaler, LoadMetrics
from ray.autoscaler.tags import TAG_RAY_NODE_TYPE, TAG_RAY_NODE_STATUS
from ray.autoscaler.node_provider import NODE_PROVIDERS, NodeProvider
from ray.autoscaler.updater import NodeUpdaterThread
@@ -21,6 +21,7 @@ class MockNode(object):
self.state = "pending"
self.tags = tags
self.external_ip = "1.2.3.4"
self.internal_ip = "172.0.0.{}".format(self.node_id)
def matches(self, tags):
for k, v in tags.items():
@@ -64,6 +65,9 @@ class MockProvider(NodeProvider):
def node_tags(self, node_id):
return self.mock_nodes[node_id].tags
def internal_ip(self, node_id):
return self.mock_nodes[node_id].internal_ip
def external_ip(self, node_id):
return self.mock_nodes[node_id].external_ip
@@ -85,6 +89,8 @@ SMALL_CLUSTER = {
"cluster_name": "default",
"min_workers": 2,
"max_workers": 2,
"target_utilization_fraction": 0.8,
"idle_timeout_minutes": 5,
"provider": {
"type": "mock",
"region": "us-east-1",
@@ -100,11 +106,55 @@ SMALL_CLUSTER = {
"TestProp": 2,
},
"file_mounts": {},
"head_init_commands": ["cmd1", "cmd2"],
"worker_init_commands": ["cmd1"],
"setup_commands": ["cmd1"],
"head_setup_commands": ["cmd2"],
"worker_setup_commands": ["cmd3"],
"head_start_ray_commands": ["start_ray_head"],
"worker_start_ray_commands": ["start_ray_worker"],
}
class LoadMetricsTest(unittest.TestCase):
def testUpdate(self):
lm = LoadMetrics()
lm.update("1.1.1.1", {"CPU": 2}, {"CPU": 1})
self.assertEqual(lm.approx_workers_used(), 0.5)
lm.update("1.1.1.1", {"CPU": 2}, {"CPU": 0})
self.assertEqual(lm.approx_workers_used(), 1.0)
lm.update("2.2.2.2", {"CPU": 2}, {"CPU": 0})
self.assertEqual(lm.approx_workers_used(), 2.0)
def testPruneByNodeIp(self):
lm = LoadMetrics()
lm.update("1.1.1.1", {"CPU": 1}, {"CPU": 0})
lm.update("2.2.2.2", {"CPU": 1}, {"CPU": 0})
lm.prune_active_ips({"1.1.1.1", "4.4.4.4"})
self.assertEqual(lm.approx_workers_used(), 1.0)
def testBottleneckResource(self):
lm = LoadMetrics()
lm.update("1.1.1.1", {"CPU": 2}, {"CPU": 0})
lm.update("2.2.2.2", {"CPU": 2, "GPU": 16}, {"CPU": 2, "GPU": 2})
self.assertEqual(lm.approx_workers_used(), 1.88)
def testHeartbeat(self):
lm = LoadMetrics()
lm.update("1.1.1.1", {"CPU": 2}, {"CPU": 1})
lm.mark_active("2.2.2.2")
self.assertIn("1.1.1.1", lm.last_heartbeat_time_by_ip)
self.assertIn("2.2.2.2", lm.last_heartbeat_time_by_ip)
self.assertNotIn("3.3.3.3", lm.last_heartbeat_time_by_ip)
def testDebugString(self):
lm = LoadMetrics()
lm.update("1.1.1.1", {"CPU": 2}, {"CPU": 0})
lm.update("2.2.2.2", {"CPU": 2, "GPU": 16}, {"CPU": 2, "GPU": 2})
debug = lm.debug_string()
self.assertIn("ResourceUsage: 2.0/4.0 CPU, 14.0/16.0 GPU", debug)
self.assertIn("NumNodesConnected: 2", debug)
self.assertIn("NumNodesUsed: 1.88", debug)
class AutoscalingTest(unittest.TestCase):
def setUp(self):
NODE_PROVIDERS["mock"] = \
@@ -137,12 +187,15 @@ class AutoscalingTest(unittest.TestCase):
def testInvalidConfig(self):
invalid_config = "/dev/null"
self.assertRaises(
ValueError, lambda: StandardAutoscaler(invalid_config))
ValueError,
lambda: StandardAutoscaler(
invalid_config, LoadMetrics(), update_interval_s=0))
def testScaleUp(self):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
autoscaler = StandardAutoscaler(config_path, max_failures=0)
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=0, update_interval_s=0)
self.assertEqual(len(self.provider.nodes({})), 0)
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
@@ -156,7 +209,8 @@ class AutoscalingTest(unittest.TestCase):
config_path = self.write_config(config)
self.provider = MockProvider()
self.provider.create_node({}, {TAG_RAY_NODE_TYPE: "Worker"}, 10)
autoscaler = StandardAutoscaler(config_path, max_failures=0)
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=0, update_interval_s=0)
self.assertEqual(len(self.provider.nodes({})), 10)
# Gradually scales down to meet target size, never going too low
@@ -172,7 +226,8 @@ class AutoscalingTest(unittest.TestCase):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
autoscaler = StandardAutoscaler(
config_path, max_concurrent_launches=5, max_failures=0)
config_path, LoadMetrics(), max_concurrent_launches=5,
max_failures=0, update_interval_s=0)
self.assertEqual(len(self.provider.nodes({})), 0)
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
@@ -185,6 +240,7 @@ class AutoscalingTest(unittest.TestCase):
self.assertEqual(len(self.provider.nodes({})), 1)
# Update the config to reduce the cluster size
new_config["min_workers"] = 10
new_config["max_workers"] = 10
self.write_config(new_config)
autoscaler.update()
@@ -192,10 +248,25 @@ class AutoscalingTest(unittest.TestCase):
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 10)
def testUpdateThrottling(self):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_concurrent_launches=5,
max_failures=0, update_interval_s=10)
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
new_config = SMALL_CLUSTER.copy()
new_config["max_workers"] = 1
self.write_config(new_config)
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2) # not updated yet
def testLaunchConfigChange(self):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
autoscaler = StandardAutoscaler(config_path, max_failures=0)
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=0, update_interval_s=0)
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
@@ -214,7 +285,8 @@ class AutoscalingTest(unittest.TestCase):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
autoscaler = StandardAutoscaler(
config_path, max_concurrent_launches=10, max_failures=0)
config_path, LoadMetrics(), max_concurrent_launches=10,
max_failures=0, update_interval_s=0)
autoscaler.update()
# Write a corrupted config
@@ -225,6 +297,7 @@ class AutoscalingTest(unittest.TestCase):
# New a good config again
new_config = SMALL_CLUSTER.copy()
new_config["min_workers"] = 10
new_config["max_workers"] = 10
self.write_config(new_config)
autoscaler.update()
@@ -234,7 +307,8 @@ class AutoscalingTest(unittest.TestCase):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
self.provider.throw = True
autoscaler = StandardAutoscaler(config_path, max_failures=2)
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=2, update_interval_s=0)
autoscaler.update()
autoscaler.update()
self.assertRaises(Exception, autoscaler.update)
@@ -243,13 +317,15 @@ class AutoscalingTest(unittest.TestCase):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
self.provider.fail_creates = True
autoscaler = StandardAutoscaler(config_path, max_failures=0)
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=0, update_interval_s=0)
self.assertRaises(AssertionError, autoscaler.update)
def testLaunchNewNodeOnOutOfBandTerminate(self):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
autoscaler = StandardAutoscaler(config_path, max_failures=0)
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=0, update_interval_s=0)
autoscaler.update()
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
@@ -264,8 +340,9 @@ class AutoscalingTest(unittest.TestCase):
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(
config_path, max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread)
config_path, LoadMetrics(), max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread,
update_interval_s=0)
autoscaler.update()
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
@@ -283,8 +360,9 @@ class AutoscalingTest(unittest.TestCase):
self.provider = MockProvider()
runner = MockProcessRunner(fail_cmds=["cmd1"])
autoscaler = StandardAutoscaler(
config_path, max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread)
config_path, LoadMetrics(), max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread,
update_interval_s=0)
autoscaler.update()
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
@@ -302,8 +380,9 @@ class AutoscalingTest(unittest.TestCase):
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(
config_path, max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread)
config_path, LoadMetrics(), max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread,
update_interval_s=0)
autoscaler.update()
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
@@ -315,12 +394,76 @@ class AutoscalingTest(unittest.TestCase):
{TAG_RAY_NODE_STATUS: "Up-to-date"})) == 2)
runner.calls = []
new_config = SMALL_CLUSTER.copy()
new_config["worker_init_commands"] = ["cmdX", "cmdY"]
new_config["worker_setup_commands"] = ["cmdX", "cmdY"]
self.write_config(new_config)
autoscaler.update()
autoscaler.update()
self.waitFor(lambda: len(runner.calls) > 0)
def testScaleUpBasedOnLoad(self):
config = SMALL_CLUSTER.copy()
config["min_workers"] = 2
config["max_workers"] = 10
config["target_utilization_fraction"] = 0.5
config_path = self.write_config(config)
self.provider = MockProvider()
lm = LoadMetrics()
autoscaler = StandardAutoscaler(
config_path, lm, max_failures=0, update_interval_s=0)
self.assertEqual(len(self.provider.nodes({})), 0)
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
# Scales up as nodes are reported as used
lm.update("172.0.0.0", {"CPU": 2}, {"CPU": 0})
lm.update("172.0.0.1", {"CPU": 2}, {"CPU": 0})
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 4)
lm.update("172.0.0.2", {"CPU": 2}, {"CPU": 0})
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 6)
# Holds steady when load is removed
lm.update("172.0.0.0", {"CPU": 2}, {"CPU": 2})
lm.update("172.0.0.1", {"CPU": 2}, {"CPU": 2})
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 6)
# Scales down as nodes become unused
lm.last_used_time_by_ip["172.0.0.0"] = 0
lm.last_used_time_by_ip["172.0.0.1"] = 0
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 4)
lm.last_used_time_by_ip["172.0.0.2"] = 0
lm.last_used_time_by_ip["172.0.0.3"] = 0
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
def testRecoverUnhealthyWorkers(self):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
runner = MockProcessRunner()
lm = LoadMetrics()
autoscaler = StandardAutoscaler(
config_path, lm, max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread,
update_interval_s=0)
autoscaler.update()
for node in self.provider.mock_nodes.values():
node.state = "running"
autoscaler.update()
self.waitFor(
lambda: len(self.provider.nodes(
{TAG_RAY_NODE_STATUS: "Up-to-date"})) == 2)
# Mark a node as unhealthy
lm.last_heartbeat_time_by_ip["172.0.0.0"] = 0
num_calls = len(runner.calls)
autoscaler.update()
self.waitFor(lambda: len(runner.calls) > num_calls)
if __name__ == "__main__":
unittest.main(verbosity=2)