mirror of
https://github.com/wassname/ray.git
synced 2026-07-22 13:00:49 +08:00
Revert "Revert "[Autoscaler] Monitor refactor for backward compatability. (#13970)" (#14046)" (#14050)
* prepare for head node
* move command runner interface outside _private
* remove space
* Eric
* flake
* min_workers in multi node type
* fixing edge cases
* eric not idle
* fix target_workers to consider min_workers of node types
* idle timeout
* minor
* minor fix
* test
* lint
* eric v2
* eric 3
* min_workers constraint before bin packing
* Update resource_demand_scheduler.py
* Revert "Update resource_demand_scheduler.py"
This reverts commit 818a63a2c86d8437b3ef21c5035d701c1d1127b5.
* reducing diff
* make get_nodes_to_launch return a dict
* merge
* weird merge fix
* auto fill instance types for AWS
* Alex/Eric
* Update doc/source/cluster/autoscaling.rst
* merge autofill and input from user
* logger.exception
* make the yaml use the default autofill
* docs Eric
* remove test_autoscaler_yaml from windows tests
* lets try changing the test a bit
* return test
* lets see
* edward
* Limit max launch concurrency
* commenting frac TODO
* move to resource demand scheduler
* use STATUS UP TO DATE
* Eric
* make logger of gc freed refs debug instead of info
* add cluster name to docker mount prefix directory
* grrR
* fix tests
* moving docker directory to sdk
* move the import to prevent circular dependency
* smallf fix
* ian
* fix max launch concurrency bug to assume failing nodes as pending and consider only load_metric's connected nodes as running
* small fix
* Revert "Revert "[Autoscaler] Monitor refactor for backward compatability. (#13970)" (#14046)"
This reverts commit 6f9d39fb3e.
* fake news
Co-authored-by: Ameer Haj Ali <ameerhajali@ameers-mbp.lan>
Co-authored-by: Alex Wu <alex@anyscale.io>
Co-authored-by: Alex Wu <itswu.alex@gmail.com>
Co-authored-by: Eric Liang <ekhliang@gmail.com>
Co-authored-by: Ameer Haj Ali <ameerhajali@Ameers-MacBook-Pro.local>
This commit is contained in:
Ameer Haj Ali
co-authored by
Ameer Haj Ali
Alex Wu
Alex Wu
Eric Liang
Ameer Haj Ali
Ameer Haj Ali
Alex Wu
Alex Wu
Eric Liang
Ameer Haj Ali
parent
c5574a33e4
commit
d87a82e891
@@ -46,19 +46,6 @@ enum TaskType {
|
||||
DRIVER_TASK = 3;
|
||||
}
|
||||
|
||||
// Type of placement group strategy.
|
||||
enum PlacementStrategy {
|
||||
// Packs Bundles into as few nodes as possible.
|
||||
PACK = 0;
|
||||
// Places Bundles across distinct nodes or processes as even as possible.
|
||||
SPREAD = 1;
|
||||
// Packs Bundles within one node. The group is not allowed to span multiple nodes.
|
||||
STRICT_PACK = 2;
|
||||
// Places Bundles across distinct nodes.
|
||||
// The group is not allowed to deploy more than one bundle on a node.
|
||||
STRICT_SPREAD = 3;
|
||||
}
|
||||
|
||||
// Address of a worker or node manager.
|
||||
message Address {
|
||||
bytes raylet_id = 1;
|
||||
@@ -456,3 +443,24 @@ enum WorkerExitType {
|
||||
// Worker exit due to placement group removal.
|
||||
PLACEMENT_GROUP_REMOVED = 3;
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/* Please do not modify/remove/change the following enum to maintain
|
||||
backwards compatibility in autoscaler. This is necessary to make sure we can
|
||||
run autoscaler with any version of ray. For example, the K8s operator runs
|
||||
autoscaler in a separate pod, if the user upgrades the ray version on the head
|
||||
pod autoscaler can crash (if the newer version of ray modified the messages
|
||||
below). */
|
||||
|
||||
// Type of placement group strategy.
|
||||
enum PlacementStrategy {
|
||||
// Packs Bundles into as few nodes as possible.
|
||||
PACK = 0;
|
||||
// Places Bundles across distinct nodes or processes as even as possible.
|
||||
SPREAD = 1;
|
||||
// Packs Bundles within one node. The group is not allowed to span multiple nodes.
|
||||
STRICT_PACK = 2;
|
||||
// Places Bundles across distinct nodes.
|
||||
// The group is not allowed to deploy more than one bundle on a node.
|
||||
STRICT_SPREAD = 3;
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
+106
-97
@@ -158,43 +158,6 @@ message ErrorTableData {
|
||||
double timestamp = 4;
|
||||
}
|
||||
|
||||
message PlacementGroupTableData {
|
||||
// State of a placement group.
|
||||
enum PlacementGroupState {
|
||||
// Placement Group is pending or scheduling
|
||||
PENDING = 0;
|
||||
// Placement Group is created.
|
||||
CREATED = 1;
|
||||
// Placement Group is already removed and won't be reschedule.
|
||||
REMOVED = 2;
|
||||
// Placement Group is rescheduling because the node it placed is dead.
|
||||
RESCHEDULING = 3;
|
||||
}
|
||||
|
||||
// ID of the PlacementGroup.
|
||||
bytes placement_group_id = 1;
|
||||
// The name of the placement group.
|
||||
string name = 2;
|
||||
// The array of the bundle in Placement Group.
|
||||
repeated Bundle bundles = 3;
|
||||
// The schedule strategy of this Placement Group.
|
||||
PlacementStrategy strategy = 4;
|
||||
// Current state of this placement group.
|
||||
PlacementGroupState state = 5;
|
||||
// Fields to detect the owner of the placement group
|
||||
// for automatic lifecycle management.
|
||||
// The job id that created this placement group.
|
||||
bytes creator_job_id = 6;
|
||||
// The actor id that created this placement group.
|
||||
bytes creator_actor_id = 7;
|
||||
// Whether or not if the creator job is dead.
|
||||
bool creator_job_dead = 8;
|
||||
// Whether or not if the creator actor is dead.
|
||||
bool creator_actor_dead = 9;
|
||||
// Whether the placement group is persistent.
|
||||
bool is_detached = 10;
|
||||
}
|
||||
|
||||
message ScheduleData {
|
||||
map<string, bytes> schedule_plan = 1;
|
||||
}
|
||||
@@ -275,71 +238,11 @@ message GcsNodeInfo {
|
||||
int64 timestamp = 10;
|
||||
}
|
||||
|
||||
// Represents the demand for a particular resource shape.
|
||||
message ResourceDemand {
|
||||
// The resource shape requested. This is a map from the resource string
|
||||
// (e.g., "CPU") to the amount requested.
|
||||
map<string, double> shape = 1;
|
||||
// The number of requests that are ready to run (i.e., dependencies have been
|
||||
// fulfilled), but that are waiting for resources.
|
||||
uint64 num_ready_requests_queued = 2;
|
||||
// The number of requests for which there is no node that is a superset of
|
||||
// the requested resource shape.
|
||||
uint64 num_infeasible_requests_queued = 3;
|
||||
// The number of requests of this shape still queued in CoreWorkers that this
|
||||
// raylet knows about.
|
||||
int64 backlog_size = 4;
|
||||
}
|
||||
|
||||
// Represents the demand sorted by resource shape.
|
||||
message ResourceLoad {
|
||||
// A list of all resource demands. The resource shape in each demand is
|
||||
// unique.
|
||||
repeated ResourceDemand resource_demands = 1;
|
||||
}
|
||||
|
||||
message PlacementGroupLoad {
|
||||
// The list of pending placement group specifications.
|
||||
repeated PlacementGroupTableData placement_group_data = 1;
|
||||
}
|
||||
|
||||
message HeartbeatTableData {
|
||||
// Node id.
|
||||
bytes node_id = 1;
|
||||
}
|
||||
|
||||
message ResourcesData {
|
||||
// Node id.
|
||||
bytes node_id = 1;
|
||||
// Resource capacity currently available on this node manager.
|
||||
map<string, double> resources_available = 2;
|
||||
// Indicates whether available resources is changed. Only used when light
|
||||
// heartbeat enabled.
|
||||
bool resources_available_changed = 3;
|
||||
// Total resource capacity configured for this node manager.
|
||||
map<string, double> resources_total = 4;
|
||||
// Aggregate outstanding resource load on this node manager.
|
||||
map<string, double> resource_load = 5;
|
||||
// Indicates whether resource load is changed. Only used when
|
||||
// light heartbeat enabled.
|
||||
bool resource_load_changed = 6;
|
||||
// The resource load on this node, sorted by resource shape.
|
||||
ResourceLoad resource_load_by_shape = 7;
|
||||
// Whether this node manager is requesting global GC.
|
||||
bool should_global_gc = 8;
|
||||
// IP address of the node.
|
||||
string node_manager_address = 9;
|
||||
}
|
||||
|
||||
message ResourceUsageBatchData {
|
||||
repeated ResourcesData batch = 1;
|
||||
// The total resource demand on all nodes included in the batch, sorted by
|
||||
// resource shape.
|
||||
ResourceLoad resource_load_by_shape = 2;
|
||||
// The pending list of placement groups.
|
||||
PlacementGroupLoad placement_group_load = 3;
|
||||
}
|
||||
|
||||
// Data for a lease on task execution.
|
||||
message TaskLeaseData {
|
||||
// The task ID.
|
||||
@@ -453,3 +356,109 @@ message PubSubMessage {
|
||||
bytes id = 1;
|
||||
bytes data = 2;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/* Please do not modify/remove/change the following messages to maintain
|
||||
backwards compatibility in autoscaler. This is necessary to make sure we can
|
||||
run autoscaler with any version of ray. For example, the K8s operator runs
|
||||
autoscaler in a separate pod, if the user upgrades the ray version on the head
|
||||
pod autoscaler can crash (if the newer version of ray modified the messages
|
||||
below). */
|
||||
|
||||
// Represents the demand for a particular resource shape.
|
||||
message ResourceDemand {
|
||||
// The resource shape requested. This is a map from the resource string
|
||||
// (e.g., "CPU") to the amount requested.
|
||||
map<string, double> shape = 1;
|
||||
// The number of requests that are ready to run (i.e., dependencies have been
|
||||
// fulfilled), but that are waiting for resources.
|
||||
uint64 num_ready_requests_queued = 2;
|
||||
// The number of requests for which there is no node that is a superset of
|
||||
// the requested resource shape.
|
||||
uint64 num_infeasible_requests_queued = 3;
|
||||
// The number of requests of this shape still queued in CoreWorkers that this
|
||||
// raylet knows about.
|
||||
int64 backlog_size = 4;
|
||||
}
|
||||
|
||||
// Represents the demand sorted by resource shape.
|
||||
message ResourceLoad {
|
||||
// A list of all resource demands. The resource shape in each demand is
|
||||
// unique.
|
||||
repeated ResourceDemand resource_demands = 1;
|
||||
}
|
||||
|
||||
message ResourcesData {
|
||||
// Node id.
|
||||
bytes node_id = 1;
|
||||
// Resource capacity currently available on this node manager.
|
||||
map<string, double> resources_available = 2;
|
||||
// Indicates whether available resources is changed. Only used when light
|
||||
// heartbeat enabled.
|
||||
bool resources_available_changed = 3;
|
||||
// Total resource capacity configured for this node manager.
|
||||
map<string, double> resources_total = 4;
|
||||
// Aggregate outstanding resource load on this node manager.
|
||||
map<string, double> resource_load = 5;
|
||||
// Indicates whether resource load is changed. Only used when
|
||||
// light heartbeat enabled.
|
||||
bool resource_load_changed = 6;
|
||||
// The resource load on this node, sorted by resource shape.
|
||||
ResourceLoad resource_load_by_shape = 7;
|
||||
// Whether this node manager is requesting global GC.
|
||||
bool should_global_gc = 8;
|
||||
// IP address of the node.
|
||||
string node_manager_address = 9;
|
||||
}
|
||||
|
||||
message ResourceUsageBatchData {
|
||||
repeated ResourcesData batch = 1;
|
||||
// The total resource demand on all nodes included in the batch, sorted by
|
||||
// resource shape.
|
||||
ResourceLoad resource_load_by_shape = 2;
|
||||
// The pending list of placement groups.
|
||||
PlacementGroupLoad placement_group_load = 3;
|
||||
}
|
||||
|
||||
message PlacementGroupLoad {
|
||||
// The list of pending placement group specifications.
|
||||
repeated PlacementGroupTableData placement_group_data = 1;
|
||||
}
|
||||
|
||||
message PlacementGroupTableData {
|
||||
// State of a placement group.
|
||||
enum PlacementGroupState {
|
||||
// Placement Group is pending or scheduling
|
||||
PENDING = 0;
|
||||
// Placement Group is created.
|
||||
CREATED = 1;
|
||||
// Placement Group is already removed and won't be reschedule.
|
||||
REMOVED = 2;
|
||||
// Placement Group is rescheduling because the node it placed is dead.
|
||||
RESCHEDULING = 3;
|
||||
}
|
||||
|
||||
// ID of the PlacementGroup.
|
||||
bytes placement_group_id = 1;
|
||||
// The name of the placement group.
|
||||
string name = 2;
|
||||
// The array of the bundle in Placement Group.
|
||||
repeated Bundle bundles = 3;
|
||||
// The schedule strategy of this Placement Group.
|
||||
PlacementStrategy strategy = 4;
|
||||
// Current state of this placement group.
|
||||
PlacementGroupState state = 5;
|
||||
// Fields to detect the owner of the placement group
|
||||
// for automatic lifecycle management.
|
||||
// The job id that created this placement group.
|
||||
bytes creator_job_id = 6;
|
||||
// The actor id that created this placement group.
|
||||
bytes creator_actor_id = 7;
|
||||
// Whether or not if the creator job is dead.
|
||||
bool creator_job_dead = 8;
|
||||
// Whether or not if the creator actor is dead.
|
||||
bool creator_actor_dead = 9;
|
||||
// Whether the placement group is persistent.
|
||||
bool is_detached = 10;
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -19,11 +19,6 @@ package ray.rpc;
|
||||
import "src/ray/protobuf/common.proto";
|
||||
import "src/ray/protobuf/gcs.proto";
|
||||
|
||||
message GcsStatus {
|
||||
int32 code = 1;
|
||||
string message = 2;
|
||||
}
|
||||
|
||||
message AddJobRequest {
|
||||
JobTableData data = 1;
|
||||
}
|
||||
@@ -231,31 +226,6 @@ message ReportResourceUsageReply {
|
||||
GcsStatus status = 1;
|
||||
}
|
||||
|
||||
message GetAllResourceUsageRequest {
|
||||
}
|
||||
|
||||
message GetAllResourceUsageReply {
|
||||
GcsStatus status = 1;
|
||||
ResourceUsageBatchData resource_usage_data = 2;
|
||||
}
|
||||
|
||||
// Service for node resource info access.
|
||||
service NodeResourceInfoGcsService {
|
||||
// Get node's resources from GCS Service.
|
||||
rpc GetResources(GetResourcesRequest) returns (GetResourcesReply);
|
||||
// Update resources of a node in GCS Service.
|
||||
rpc UpdateResources(UpdateResourcesRequest) returns (UpdateResourcesReply);
|
||||
// Delete resources of a node in GCS Service.
|
||||
rpc DeleteResources(DeleteResourcesRequest) returns (DeleteResourcesReply);
|
||||
// Get available resources of all nodes.
|
||||
rpc GetAllAvailableResources(GetAllAvailableResourcesRequest)
|
||||
returns (GetAllAvailableResourcesReply);
|
||||
// Report resource usage of a node to GCS Service.
|
||||
rpc ReportResourceUsage(ReportResourceUsageRequest) returns (ReportResourceUsageReply);
|
||||
// Get resource usage of all nodes from GCS Service.
|
||||
rpc GetAllResourceUsage(GetAllResourceUsageRequest) returns (GetAllResourceUsageReply);
|
||||
}
|
||||
|
||||
// Service for heartbeat info access.
|
||||
service HeartbeatInfoGcsService {
|
||||
// Report heartbeat of a node to GCS Service.
|
||||
@@ -553,3 +523,41 @@ service PlacementGroupInfoGcsService {
|
||||
rpc WaitPlacementGroupUntilReady(WaitPlacementGroupUntilReadyRequest)
|
||||
returns (WaitPlacementGroupUntilReadyReply);
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/* Please do not modify/remove/change the following messages to maintain
|
||||
backwards compatibility in autoscaler. This is necessary to make sure we can
|
||||
run autoscaler with any version of ray. For example, the K8s operator runs
|
||||
autoscaler in a separate pod, if the user upgrades the ray version on the head
|
||||
pod autoscaler can crash (if the newer version of ray modified the messages
|
||||
below). */
|
||||
|
||||
message GetAllResourceUsageRequest {
|
||||
}
|
||||
|
||||
message GetAllResourceUsageReply {
|
||||
GcsStatus status = 1;
|
||||
ResourceUsageBatchData resource_usage_data = 2;
|
||||
}
|
||||
|
||||
// Service for node resource info access.
|
||||
service NodeResourceInfoGcsService {
|
||||
// Get node's resources from GCS Service.
|
||||
rpc GetResources(GetResourcesRequest) returns (GetResourcesReply);
|
||||
// Update resources of a node in GCS Service.
|
||||
rpc UpdateResources(UpdateResourcesRequest) returns (UpdateResourcesReply);
|
||||
// Delete resources of a node in GCS Service.
|
||||
rpc DeleteResources(DeleteResourcesRequest) returns (DeleteResourcesReply);
|
||||
// Get available resources of all nodes.
|
||||
rpc GetAllAvailableResources(GetAllAvailableResourcesRequest)
|
||||
returns (GetAllAvailableResourcesReply);
|
||||
// Report resource usage of a node to GCS Service.
|
||||
rpc ReportResourceUsage(ReportResourceUsageRequest) returns (ReportResourceUsageReply);
|
||||
// Get resource usage of all nodes from GCS Service.
|
||||
rpc GetAllResourceUsage(GetAllResourceUsageRequest) returns (GetAllResourceUsageReply);
|
||||
}
|
||||
|
||||
message GcsStatus {
|
||||
int32 code = 1;
|
||||
string message = 2;
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Reference in New Issue
Block a user