wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-14 11:17:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Updating zero capacity resource semantics (#4555)

Browse Source
This commit is contained in:
Romil Bhardwaj
2019-04-12 16:53:57 -07:00
committed by Robert Nishihara
parent bb207a205b
commit 0f42f87ebc
17 changed files with 248 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/raylet/node_manager.cc
+19 -14
View File
@@ -1469,15 +1469,16 @@ void NodeManager::HandleTaskBlocked(const std::shared_ptr<LocalClientConnection>
// Get the CPU resources required by the running task.
const auto required_resources = task.GetTaskSpecification().GetRequiredResources();
double required_cpus = required_resources.GetNumCpus();
const std::unordered_map<std::string, double> cpu_resources = {
{kCPU_ResourceLabel, required_cpus}};
std::unordered_map<std::string, double> cpu_resources;
if (required_cpus > 0) {
cpu_resources[kCPU_ResourceLabel] = required_cpus;
}
// Release the CPU resources.
auto const cpu_resource_ids = worker->ReleaseTaskCpuResources();
local_available_resources_.Release(cpu_resource_ids);
RAY_CHECK(
cluster_resource_map_[gcs_client_->client_table().GetLocalClientId()].Release(
ResourceSet(cpu_resources)));
cluster_resource_map_[gcs_client_->client_table().GetLocalClientId()].Release(
ResourceSet(cpu_resources));
worker->MarkBlocked();
// Try dispatching tasks since we may have released some resources.
@@ -1521,9 +1522,11 @@ void NodeManager::HandleTaskUnblocked(
// Get the CPU resources required by the running task.
const auto required_resources = task.GetTaskSpecification().GetRequiredResources();
double required_cpus = required_resources.GetNumCpus();
const ResourceSet cpu_resources(
std::unordered_map<std::string, double>({{kCPU_ResourceLabel, required_cpus}}));
std::unordered_map<std::string, double> cpu_resources_map;
if (required_cpus > 0) {
cpu_resources_map[kCPU_ResourceLabel] = required_cpus;
}
const ResourceSet cpu_resources(cpu_resources_map);
// Check if we can reacquire the CPU resources.
bool oversubscribed = !local_available_resources_.Contains(cpu_resources);
@@ -1533,9 +1536,8 @@ void NodeManager::HandleTaskUnblocked(
// reacquire here may be different from the ones that the task started with.
auto const resource_ids = local_available_resources_.Acquire(cpu_resources);
worker->AcquireTaskCpuResources(resource_ids);
RAY_CHECK(
cluster_resource_map_[gcs_client_->client_table().GetLocalClientId()].Acquire(
cpu_resources));
cluster_resource_map_[gcs_client_->client_table().GetLocalClientId()].Acquire(
cpu_resources);
} else {
// In this case, we simply don't reacquire the CPU resources for the worker.
// The worker can keep running and when the task finishes, it will simply
@@ -1627,7 +1629,7 @@ bool NodeManager::AssignTask(const Task &task) {
auto acquired_resources =
local_available_resources_.Acquire(spec.GetRequiredResources());
const auto &my_client_id = gcs_client_->client_table().GetLocalClientId();
RAY_CHECK(cluster_resource_map_[my_client_id].Acquire(spec.GetRequiredResources()));
cluster_resource_map_[my_client_id].Acquire(spec.GetRequiredResources());
if (spec.IsActorCreationTask()) {
// Check that we are not placing an actor creation task on a node with 0 CPUs.
@@ -1741,8 +1743,8 @@ void NodeManager::FinishAssignedTask(Worker &worker) {
// Release task's resources. The worker's lifetime resources are still held.
auto const &task_resources = worker.GetTaskResourceIds();
local_available_resources_.Release(task_resources);
RAY_CHECK(cluster_resource_map_[gcs_client_->client_table().GetLocalClientId()].Release(
task_resources.ToResourceSet()));
cluster_resource_map_[gcs_client_->client_table().GetLocalClientId()].Release(
task_resources.ToResourceSet());
worker.ResetTaskResourceIds();
// If this was an actor or actor creation task, handle the actor's new state.
@@ -2034,6 +2036,9 @@ void NodeManager::ForwardTaskOrResubmit(const Task &task,
RAY_LOG(INFO) << "Failed to forward task " << task_id << " to node manager "
<< node_manager_id;
// TODO(romilb): We should probably revert the load subtraction from
// SchedulingPolicy::Schedule()
// Mark the failed task as pending to let other raylets know that we still
// have the task. TaskDependencyManager::TaskPending() is assumed to be
// idempotent.
src/ray/raylet/scheduling_policy.cc
+5 -3
View File
@@ -49,11 +49,13 @@ std::unordered_map<TaskID, ClientID> SchedulingPolicy::Schedule(
const auto &node_resources = client_resource_pair.second;
ResourceSet available_node_resources =
ResourceSet(node_resources.GetAvailableResources());
available_node_resources.SubtractResourcesStrict(node_resources.GetLoadResources());
// TODO(romilb): Why do we need to subtract load from available resources?
// Even if we don't the code path below for choosing a dst_client_id would be
// similar.
available_node_resources.SubtractResources(node_resources.GetLoadResources());
RAY_LOG(DEBUG) << "client_id " << node_client_id
<< " avail: " << node_resources.GetAvailableResources().ToString()
<< " load: " << node_resources.GetLoadResources().ToString()
<< " avail-load: " << available_node_resources.ToString();
<< " load: " << node_resources.GetLoadResources().ToString();
if (resource_demand.IsSubset(available_node_resources)) {
// This node is a feasible candidate.
src/ray/raylet/scheduling_resources.cc
+94 -93
View File
@@ -9,16 +9,39 @@ namespace ray {
namespace raylet {
bool EqualsZeroEpsilon(double quantity) {
if ((quantity <= EPSILON) && (quantity >= -1 * EPSILON)) {
return true;
}
return false;
}
bool EqualsOneEpsilon(double quantity) {
if ((quantity <= 1 + EPSILON) && (quantity >= 1 - EPSILON)) {
return true;
}
return false;
}
ResourceSet::ResourceSet() {}
ResourceSet::ResourceSet(const std::unordered_map<std::string, double> &resource_map)
: resource_capacity_(resource_map) {}
: resource_capacity_(resource_map) {
for (auto const &resource_pair : resource_capacity_) {
RAY_CHECK(resource_pair.second > 0 + EPSILON)
<< "Resource " << resource_pair.first << " capacity is " << resource_pair.second
<< ". Should have been greater than zero.";
}
}
ResourceSet::ResourceSet(const std::vector<std::string> &resource_labels,
const std::vector<double> resource_capacity) {
RAY_CHECK(resource_labels.size() == resource_capacity.size());
for (uint i = 0; i < resource_labels.size(); i++) {
RAY_CHECK(AddResource(resource_labels[i], resource_capacity[i]));
RAY_CHECK(resource_capacity[i] > 0 + EPSILON)
<< "Resource " << resource_labels[i] << " capacity is " << resource_capacity[i]
<< ". Should have been greater than zero.";
resource_capacity_[resource_labels[i]] = resource_capacity[i];
}
}
@@ -30,13 +53,7 @@ bool ResourceSet::operator==(const ResourceSet &rhs) const {
bool ResourceSet::IsEmpty() const {
// Check whether the capacity of each resource type is zero. Exit early if not.
if (resource_capacity_.empty()) return true;
for (const auto &resource_pair : resource_capacity_) {
if (resource_pair.second > 0) {
return false;
}
}
return true;
return resource_capacity_.empty();
}
bool ResourceSet::IsSubset(const ResourceSet &other) const {
@@ -44,11 +61,7 @@ bool ResourceSet::IsSubset(const ResourceSet &other) const {
for (const auto &resource_pair : resource_capacity_) {
const auto &resource_name = resource_pair.first;
const double lhs_quantity = resource_pair.second;
double rhs_quantity = 0;
if (!other.GetResource(resource_name, &rhs_quantity)) {
// Resource not found in rhs, therefore lhs is not a subset of rhs.
return false;
}
double rhs_quantity = other.GetResource(resource_name);
if (lhs_quantity > rhs_quantity) {
// Resource found in rhs, but lhs capacity exceeds rhs capacity.
return false;
@@ -67,41 +80,42 @@ bool ResourceSet::IsEqual(const ResourceSet &rhs) const {
return (this->IsSubset(rhs) && rhs.IsSubset(*this));
}
bool ResourceSet::AddResource(const std::string &resource_name, double capacity) {
resource_capacity_[resource_name] = capacity;
return true;
}
bool ResourceSet::RemoveResource(const std::string &resource_name) {
throw std::runtime_error("Method not implemented");
}
void ResourceSet::SubtractResources(const ResourceSet &other) {
// Subtract the resources and delete any if new capacity is zero.
for (const auto &resource_pair : other.GetResourceMap()) {
const std::string &resource_label = resource_pair.first;
const double &resource_capacity = resource_pair.second;
if (resource_capacity_.count(resource_label) == 1) {
resource_capacity_[resource_label] -= resource_capacity;
if (resource_capacity_[resource_label] < 0 + EPSILON) {
resource_capacity_.erase(resource_label);
}
}
}
}
bool ResourceSet::SubtractResourcesStrict(const ResourceSet &other) {
// Subtract the resources and track whether a resource goes below zero.
bool oversubscribed = false;
void ResourceSet::SubtractResourcesStrict(const ResourceSet &other) {
// Subtract the resources, make sure none goes below zero and delete any if new capacity
// is zero.
for (const auto &resource_pair : other.GetResourceMap()) {
const std::string &resource_label = resource_pair.first;
const double &resource_capacity = resource_pair.second;
RAY_CHECK(resource_capacity_.count(resource_label) == 1)
<< "Attempt to acquire unknown resource: " << resource_label;
resource_capacity_[resource_label] -= resource_capacity;
if (resource_capacity_[resource_label] < 0) {
oversubscribed = true;
// TODO(romilb): Double precision subtraction may sometimes be less than zero by a
// small epsilon - need to fix.
RAY_CHECK(resource_capacity_[resource_label] >= 0 - EPSILON)
<< "Capacity of resource " << resource_label << " after subtraction is negative ("
<< resource_capacity_[resource_label] << ")."
<< " Debug: resource_capacity_:" << ToString() << ", other: " << other.ToString();
if (EqualsZeroEpsilon(resource_capacity_[resource_label])) {
resource_capacity_.erase(resource_label);
}
}
return !oversubscribed;
}
// Perform a left join.
bool ResourceSet::AddResourcesStrict(const ResourceSet &other) {
// Return failure if attempting to perform vector addition with unknown labels.
for (const auto &resource_pair : other.GetResourceMap()) {
const std::string &resource_label = resource_pair.first;
const double &resource_capacity = resource_pair.second;
RAY_CHECK(resource_capacity_.count(resource_label) != 0);
resource_capacity_[resource_label] += resource_capacity;
}
return true;
}
// Perform an outer join.
@@ -109,33 +123,22 @@ void ResourceSet::AddResources(const ResourceSet &other) {
for (const auto &resource_pair : other.GetResourceMap()) {
const std::string &resource_label = resource_pair.first;
const double &resource_capacity = resource_pair.second;
if (resource_capacity_.count(resource_label) == 0) {
// Add the new label if not found.
RAY_CHECK(AddResource(resource_label, resource_capacity));
} else {
// Increment the resource by its capacity.
resource_capacity_[resource_label] += resource_capacity;
}
resource_capacity_[resource_label] += resource_capacity;
}
}
bool ResourceSet::GetResource(const std::string &resource_name, double *value) const {
if (!value) {
return false;
double ResourceSet::GetResource(const std::string &resource_name) const {
if (EqualsZeroEpsilon(resource_capacity_.count(resource_name))) {
return 0;
}
if (resource_capacity_.count(resource_name) == 0) {
*value = std::nan("");
return false;
}
*value = resource_capacity_.at(resource_name);
return true;
double capacity = resource_capacity_.at(resource_name);
RAY_CHECK(capacity > 0 + EPSILON) << "Resource " << resource_name << " capacity is "
<< capacity
<< ". Should have been greater than zero.";
return capacity;
}
double ResourceSet::GetNumCpus() const {
double num_cpus;
RAY_CHECK(GetResource(kCPU_ResourceLabel, &num_cpus));
return num_cpus;
}
double ResourceSet::GetNumCpus() const { return GetResource(kCPU_ResourceLabel); }
const std::string ResourceSet::ToString() const {
std::string return_string = "";
@@ -222,6 +225,12 @@ ResourceIds ResourceIds::Acquire(double resource_quantity) {
if (fractional_pair.second >= resource_quantity) {
auto return_pair = std::make_pair(fractional_pair.first, resource_quantity);
fractional_pair.second -= resource_quantity;
// Remove the fractional pair if the new capacity is 0
if (EqualsZeroEpsilon(fractional_pair.second)) {
std::swap(fractional_pair, fractional_ids_[fractional_ids_.size() - 1]);
fractional_ids_.pop_back();
}
return ResourceIds({return_pair});
}
}
@@ -257,15 +266,18 @@ void ResourceIds::Release(const ResourceIds &resource_ids) {
if (fractional_pair_it == fractional_ids_.end()) {
fractional_ids_.push_back(fractional_pair_to_return);
} else {
RAY_CHECK(fractional_pair_it->second < 1);
fractional_pair_it->second += fractional_pair_to_return.second;
// TODO(romilb): Double precision addition may sometimes exceed 1 by a small epsilon
// - need to fix this.
RAY_CHECK(fractional_pair_it->second <= 1 + EPSILON)
<< "Fractional Resource Id " << fractional_pair_it->first << " capacity is "
<< fractional_pair_it->second << ". Should have been less than one.";
// If this makes the ID whole, then return it to the list of whole IDs.
if (fractional_pair_it->second >= 1) {
// TODO(romilb): Double precision addition may sometimes exceed 1 by a small epsilon
// - need to fix this.
if (EqualsOneEpsilon(fractional_pair_it->second)) {
whole_ids_.push_back(resource_id);
fractional_pair_it->second -= 1;
if (fractional_pair_it->second < 1e-6) {
fractional_ids_.erase(fractional_pair_it);
}
fractional_ids_.erase(fractional_pair_it);
}
}
}
@@ -283,6 +295,10 @@ const std::vector<std::pair<int64_t, double>> &ResourceIds::FractionalIds() cons
return fractional_ids_;
}
bool ResourceIds::TotalQuantityIsZero() const {
return whole_ids_.empty() && fractional_ids_.empty();
}
double ResourceIds::TotalQuantity() const {
double total_quantity = whole_ids_.size();
for (auto const &fractional_pair : fractional_ids_) {
@@ -330,9 +346,9 @@ bool ResourceIdSet::Contains(const ResourceSet &resource_set) const {
for (auto const &resource_pair : resource_set.GetResourceMap()) {
auto const &resource_name = resource_pair.first;
double resource_quantity = resource_pair.second;
if (resource_quantity == 0) {
continue;
}
RAY_CHECK(resource_quantity > 0 + EPSILON) << "Resource " << resource_name
<< " capacity is " << resource_quantity
<< ". Should have been greater than zero.";
auto it = available_resources_.find(resource_name);
if (it == available_resources_.end()) {
@@ -352,14 +368,16 @@ ResourceIdSet ResourceIdSet::Acquire(const ResourceSet &resource_set) {
for (auto const &resource_pair : resource_set.GetResourceMap()) {
auto const &resource_name = resource_pair.first;
double resource_quantity = resource_pair.second;
if (resource_quantity == 0) {
continue;
}
RAY_CHECK(resource_quantity > 0 + EPSILON) << "Resource " << resource_name
<< " capacity is " << resource_quantity
<< ". Should have been greater than zero.";
auto it = available_resources_.find(resource_name);
RAY_CHECK(it != available_resources_.end());
acquired_resources[resource_name] = it->second.Acquire(resource_quantity);
if (it->second.TotalQuantityIsZero()) {
available_resources_.erase(it);
}
}
return ResourceIdSet(acquired_resources);
}
@@ -368,15 +386,10 @@ void ResourceIdSet::Release(const ResourceIdSet &resource_id_set) {
for (auto const &resource_pair : resource_id_set.AvailableResources()) {
auto const &resource_name = resource_pair.first;
auto const &resource_ids = resource_pair.second;
if (resource_ids.TotalQuantity() == 0) {
continue;
}
RAY_CHECK(!resource_ids.TotalQuantityIsZero());
auto it = available_resources_.find(resource_name);
if (it == available_resources_.end()) {
// This should not happen when Release is called on resources that were obtained
// through a corresponding call to Acquire.
available_resources_[resource_name] = resource_ids;
} else {
it->second.Release(resource_ids);
@@ -474,18 +487,6 @@ SchedulingResources::SchedulingResources(const ResourceSet &total)
SchedulingResources::~SchedulingResources() {}
ResourceAvailabilityStatus SchedulingResources::CheckResourcesSatisfied(
ResourceSet &resources) const {
if (!resources.IsSubset(resources_total_)) {
return ResourceAvailabilityStatus::kInfeasible;
}
// Resource demand specified is feasible. Check if it's available.
if (!resources.IsSubset(resources_available_)) {
return ResourceAvailabilityStatus::kResourcesUnavailable;
}
return ResourceAvailabilityStatus::kFeasible;
}
const ResourceSet &SchedulingResources::GetAvailableResources() const {
return resources_available_;
}
@@ -507,13 +508,13 @@ const ResourceSet &SchedulingResources::GetLoadResources() const {
}
// Return specified resources back to SchedulingResources.
bool SchedulingResources::Release(const ResourceSet &resources) {
return resources_available_.AddResourcesStrict(resources);
void SchedulingResources::Release(const ResourceSet &resources) {
resources_available_.AddResources(resources);
}
// Take specified resources from SchedulingResources.
bool SchedulingResources::Acquire(const ResourceSet &resources) {
return resources_available_.SubtractResourcesStrict(resources);
void SchedulingResources::Acquire(const ResourceSet &resources) {
resources_available_.SubtractResourcesStrict(resources);
}
std::string SchedulingResources::DebugString() const {
src/ray/raylet/scheduling_resources.h
+39 -44
View File
@@ -8,19 +8,25 @@
#include "ray/raylet/format/node_manager_generated.h"
#define EPSILON 0.00001
namespace ray {
namespace raylet {
const std::string kCPU_ResourceLabel = "CPU";
/// Resource availability status reports whether the resource requirement is
/// (1) infeasible, (2) feasible but currently unavailable, or (3) available.
enum class ResourceAvailabilityStatus : int {
kInfeasible, ///< Cannot ever satisfy resource requirements.
kResourcesUnavailable, ///< Feasible, but not currently available.
kFeasible ///< Feasible and currently available.
};
/// \brief Test if the quantity is within epsilon bounds of 0.
///
/// \param quantity: Quantity to check
/// \return True if -epsilon <= Quantity <= epsilon, False otherwise.
bool EqualsZeroEpsilon(double quantity);
/// \brief Test if the quantity is within epsilon bounds of 1.
///
/// \param quantity: Quantity to check
/// \return True if 1 - epsilon <= Quantity <= 1 + epsilon, False otherwise.
bool EqualsOneEpsilon(double quantity);
/// \class ResourceSet
/// \brief Encapsulates and operates on a set of resources, including CPUs,
@@ -67,26 +73,12 @@ class ResourceSet {
/// False otherwise.
bool IsSuperset(const ResourceSet &other) const;
/// \brief Add a new resource to the resource set.
///
/// \param resource_name: name/label of the resource to add.
/// \param capacity: numeric capacity value for the resource to add.
/// \return True, if the resource was successfully added. False otherwise.
bool AddResource(const std::string &resource_name, double capacity);
/// \brief Remove the specified resource from the resource set.
///
/// \param resource_name: name/label of the resource to remove.
/// \return True, if the resource was successfully removed. False otherwise.
bool RemoveResource(const std::string &resource_name);
/// \brief Add a set of resources to the current set of resources only if the resource
/// labels match.
///
/// \param other: The other resource set to add.
/// \return True if the resource set was added successfully. False otherwise.
bool AddResourcesStrict(const ResourceSet &other);
/// \brief Aggregate resources from the other set into this set, adding any missing
/// resource labels to this set.
///
@@ -94,21 +86,28 @@ class ResourceSet {
/// \return Void.
void AddResources(const ResourceSet &other);
/// \brief Subtract a set of resources from the current set of resources, only if
/// resource labels match.
/// \brief Subtract a set of resources from the current set of resources.
/// Deletes any resource if the capacity after subtraction is zero or negative.
///
/// \param other: The resource set to subtract from the current resource set.
/// \return True if the resource set was subtracted successfully.
/// False otherwise.
bool SubtractResourcesStrict(const ResourceSet &other);
/// \return Void.
void SubtractResources(const ResourceSet &other);
/// \brief Subtract a set of resources from the current set of resources and
/// check that the post-subtraction result nonnegative. Assumes other
/// is a subset of the ResourceSet. Deletes any resource if the capacity after
/// subtraction is zero.
///
/// \param other: The resource set to subtract from the current resource set.
/// \return Void.
void SubtractResourcesStrict(const ResourceSet &other);
/// Return the capacity value associated with the specified resource.
///
/// \param resource_name: Resource name for which capacity is requested.
/// \param[out] value: Resource capacity value.
/// \return True if the resource capacity value was successfully retrieved.
/// False otherwise.
bool GetResource(const std::string &resource_name, double *value) const;
/// \return The capacity value associated with the specified resource, zero if resource
/// does not exist.
double GetResource(const std::string &resource_name) const;
/// Return the number of CPUs.
///
@@ -126,7 +125,7 @@ class ResourceSet {
const std::string ToString() const;
private:
/// Resource capacity map.
/// Resource capacity map. The capacities (double) are always positive.
std::unordered_map<std::string, double> resource_capacity_;
};
@@ -205,6 +204,11 @@ class ResourceIds {
/// \return The fractional IDs.
const std::vector<std::pair<int64_t, double>> &FractionalIds() const;
/// \brief Check if ResourceIds has any resources.
///
/// \return True if there are no whole or fractional resources. False otherwise.
bool TotalQuantityIsZero() const;
/// \brief Return the total quantity of resources, ignoring the specific IDs.
///
/// \return The total quantity of the resource.
@@ -329,13 +333,6 @@ class SchedulingResources {
/// \brief SchedulingResources destructor.
~SchedulingResources();
/// \brief Check if the specified resource request can be satisfied.
///
/// \param set: The set of resources representing the resource request.
/// \return Availability status that specifies if the requested resource set
/// is feasible, infeasible, or feasible but unavailable.
ResourceAvailabilityStatus CheckResourcesSatisfied(ResourceSet &set) const;
/// \brief Request the set and capacity of resources currently available.
///
/// \return Immutable set of resources with currently available capacity.
@@ -363,16 +360,14 @@ class SchedulingResources {
/// \brief Release the amount of resources specified.
///
/// \param resources: the amount of resources to be released.
/// \return True if resources were successfully released. False otherwise.
bool Release(const ResourceSet &resources);
/// \return Void.
void Release(const ResourceSet &resources);
/// \brief Acquire the amount of resources specified.
///
/// \param resources: the amount of resources to be acquired.
/// \return True if resources were acquired without oversubscription. If this
/// returns false, then the resources were still acquired, but we are now at
/// negative resources.
bool Acquire(const ResourceSet &resources);
/// \return Void.
void Acquire(const ResourceSet &resources);
/// Returns debug string for class.
///
src/ray/raylet/worker_pool_test.cc
+1 -1
View File
@@ -77,7 +77,7 @@ static inline TaskSpecification ExampleTaskSpec(
std::vector<std::string> function_descriptor(3);
return TaskSpecification(DriverID::nil(), TaskID::nil(), 0, ActorID::nil(),
ObjectID::nil(), 0, actor_id, ActorHandleID::nil(), 0, {}, {},
0, {{}}, {{}}, language, function_descriptor);
0, {}, {}, language, function_descriptor);
}
TEST_F(WorkerPoolTest, HandleWorkerRegistration) {