wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-12 12:51:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

[autoscaler] Make placement groups bypass max launch limit (#13089)

Browse Source
This commit is contained in:
Ameer Haj Ali
2020-12-29 10:06:11 -08:00
committed by GitHub
parent 5a4e50c9d9
commit 44483f465c
2 changed files with 139 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/autoscaler/_private/resource_demand_scheduler.py
+34 -11
View File
@@ -159,11 +159,15 @@ class ResourceDemandScheduler:
node_resources, node_type_counts, self.node_types,
self.max_workers, self.head_node_type, ensure_min_cluster_size)
# Step 3: add nodes for strict spread groups
logger.debug(f"Placement group demands: {pending_placement_groups}")
# Step 3: get resource demands of placement groups and return the
# groups that should be strictly spread.
logger.info(f"Placement group demands: {pending_placement_groups}")
placement_group_demand_vector, strict_spreads = \
placement_groups_to_resource_demands(pending_placement_groups)
resource_demands.extend(placement_group_demand_vector)
# Place placement groups demand vector at the beginning of the resource
# demands vector to make it consistent (results in the same types of
# nodes to add) with pg_demands_nodes_max_launch_limit calculated later
resource_demands = placement_group_demand_vector + resource_demands
if self.is_legacy_yaml() and \
not self.node_types[NODE_TYPE_LEGACY_WORKER]["resources"]:
@@ -179,18 +183,32 @@ class ResourceDemandScheduler:
return self._legacy_worker_node_to_launch(
nodes, launching_nodes, node_resources,
resource_demands + request_resources_demands)
placement_group_nodes_to_add, node_resources, node_type_counts = \
spread_pg_nodes_to_add, node_resources, node_type_counts = \
self.reserve_and_allocate_spread(
strict_spreads, node_resources, node_type_counts)
# Calculate the nodes to add for bypassing max launch limit for
# placement groups and spreads.
unfulfilled_placement_groups_demands, _ = get_bin_pack_residual(
node_resources, placement_group_demand_vector)
# Add 1 to account for the head node.
max_to_add = self.max_workers + 1 - sum(node_type_counts.values())
pg_demands_nodes_max_launch_limit = get_nodes_for(
self.node_types, node_type_counts, self.head_node_type, max_to_add,
unfulfilled_placement_groups_demands)
placement_groups_nodes_max_limit = {
node_type: spread_pg_nodes_to_add.get(node_type, 0) +
pg_demands_nodes_max_launch_limit.get(node_type, 0)
for node_type in self.node_types
}
# Step 4/5: add nodes for pending tasks, actors, and non-strict spread
# groups
unfulfilled, _ = get_bin_pack_residual(node_resources,
resource_demands)
logger.debug("Resource demands: {}".format(resource_demands))
logger.debug("Unfulfilled demands: {}".format(unfulfilled))
# Add 1 to account for the head node.
max_to_add = self.max_workers + 1 - sum(node_type_counts.values())
nodes_to_add_based_on_demand = get_nodes_for(
self.node_types, node_type_counts, self.head_node_type, max_to_add,
unfulfilled)
@@ -201,15 +219,16 @@ class ResourceDemandScheduler:
for node_type in self.node_types:
nodes_to_add = (adjusted_min_workers.get(
node_type, 0) + placement_group_nodes_to_add.get(node_type, 0)
+ nodes_to_add_based_on_demand.get(node_type, 0))
node_type, 0) + spread_pg_nodes_to_add.get(node_type, 0) +
nodes_to_add_based_on_demand.get(node_type, 0))
if nodes_to_add > 0:
total_nodes_to_add[node_type] = nodes_to_add
# Limit the number of concurrent launches
total_nodes_to_add = self._get_concurrent_resource_demand_to_launch(
total_nodes_to_add, unused_resources_by_ip.keys(), nodes,
launching_nodes, adjusted_min_workers)
launching_nodes, adjusted_min_workers,
placement_groups_nodes_max_limit)
logger.debug("Node requests: {}".format(total_nodes_to_add))
return total_nodes_to_add
@@ -288,6 +307,7 @@ class ResourceDemandScheduler:
non_terminated_nodes: List[NodeID],
pending_launches_nodes: Dict[NodeType, int],
adjusted_min_workers: Dict[NodeType, int],
placement_group_nodes: Dict[NodeType, int],
) -> Dict[NodeType, int]:
"""Updates the max concurrent resources to launch for each node type.
@@ -311,6 +331,8 @@ class ResourceDemandScheduler:
min_workers and request_resources(). This overrides the launch
limits since the user is hinting to immediately scale up to
this size.
placement_group_nodes: Nodes to launch for placement groups.
This overrides the launch concurrency limits.
Returns:
Dict[NodeType, int]: Maximum number of nodes to launch for each
node type.
@@ -333,8 +355,9 @@ class ResourceDemandScheduler:
max_allowed_pending_nodes - total_pending_nodes,
# Allow more nodes if this is to respect min_workers or
# request_resources().
adjusted_min_workers.get(node_type, 0))
# request_resources() or placement groups.
adjusted_min_workers.get(node_type, 0) +
placement_group_nodes.get(node_type, 0))
if upper_bound > 0:
updated_nodes_to_launch[node_type] = min(