diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index ca32165ab..f4a4c39c3 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -118,6 +118,7 @@ void ObjectManager::HandleObjectAdded( RAY_LOG(DEBUG) << "Object added " << object_id; RAY_CHECK(local_objects_.count(object_id) == 0); local_objects_[object_id].object_info = object_info; + used_memory_ += object_info.data_size + object_info.metadata_size; ray::Status status = object_directory_->ReportObjectAdded(object_id, self_node_id_, object_info); @@ -146,6 +147,8 @@ void ObjectManager::NotifyDirectoryObjectDeleted(const ObjectID &object_id) { RAY_CHECK(it != local_objects_.end()); auto object_info = it->second.object_info; local_objects_.erase(it); + used_memory_ -= object_info.data_size + object_info.metadata_size; + RAY_CHECK(!local_objects_.empty() || used_memory_ == 0); ray::Status status = object_directory_->ReportObjectRemoved(object_id, self_node_id_, object_info); } @@ -902,24 +905,13 @@ std::string ObjectManager::DebugString() const { } void ObjectManager::RecordMetrics() const { - int64_t used_memory = 0; - for (const auto &it : local_objects_) { - object_manager::protocol::ObjectInfoT object_info = it.second.object_info; - used_memory += object_info.data_size + object_info.metadata_size; - } - stats::ObjectManagerStats().Record(used_memory, - {{stats::ValueTypeKey, "used_object_store_memory"}}); - stats::ObjectManagerStats().Record(local_objects_.size(), - {{stats::ValueTypeKey, "num_local_objects"}}); - stats::ObjectManagerStats().Record(active_wait_requests_.size(), - {{stats::ValueTypeKey, "num_active_wait_requests"}}); - stats::ObjectManagerStats().Record( - unfulfilled_push_requests_.size(), - {{stats::ValueTypeKey, "num_unfulfilled_push_requests"}}); - stats::ObjectManagerStats().Record(pull_requests_.size(), - {{stats::ValueTypeKey, "num_pull_requests"}}); - stats::ObjectManagerStats().Record(profile_events_.size(), - {{stats::ValueTypeKey, "num_profile_events"}}); + stats::ObjectStoreAvailableMemory().Record(config_.object_store_memory - used_memory_); + stats::ObjectStoreUsedMemory().Record(used_memory_); + stats::ObjectStoreLocalObjects().Record(local_objects_.size()); + stats::ObjectManagerWaitRequests().Record(active_wait_requests_.size()); + stats::ObjectManagerPullRequests().Record(pull_requests_.size()); + stats::ObjectManagerUnfulfilledPushRequests().Record(unfulfilled_push_requests_.size()); + stats::ObjectManagerProfileEvents().Record(profile_events_.size()); } } // namespace ray diff --git a/src/ray/object_manager/object_manager.h b/src/ray/object_manager/object_manager.h index 2b7142bdd..80396e840 100644 --- a/src/ray/object_manager/object_manager.h +++ b/src/ray/object_manager/object_manager.h @@ -470,6 +470,9 @@ class ObjectManager : public ObjectManagerInterface, /// Client id - object manager gRPC client. std::unordered_map> remote_object_manager_clients_; + + /// Running sum of the amount of memory used in the object store. + int64_t used_memory_ = 0; }; } // namespace ray diff --git a/src/ray/stats/metric_defs.h b/src/ray/stats/metric_defs.h index bc7b7666e..c4227a8f0 100644 --- a/src/ray/stats/metric_defs.h +++ b/src/ray/stats/metric_defs.h @@ -61,9 +61,33 @@ static Gauge RestartingActors("restarting_actors", "Number of restarting actors. static Gauge DeadActors("dead_actors", "Number of dead actors.", "actors"); -static Gauge ObjectManagerStats("object_manager_stats", - "Stat the metric values of object in raylet", "pcs", - {ValueTypeKey}); +static Gauge ObjectStoreAvailableMemory( + "object_store_available_memory", + "Amount of memory currently available in the object store.", "bytes"); + +static Gauge ObjectStoreUsedMemory( + "object_store_used_memory", + "Amount of memory currently occupied in the object store.", "bytes"); + +static Gauge ObjectStoreLocalObjects("object_store_num_local_objects", + "Number of objects currently in the object store.", + "objects"); + +static Gauge ObjectManagerWaitRequests("object_manager_num_wait_requests", + "Number of pending wait requests for objects.", + "requests"); + +static Gauge ObjectManagerPullRequests("object_manager_num_pull_requests", + "Number of active pull requests for objects.", + "requests"); + +static Gauge ObjectManagerUnfulfilledPushRequests( + "object_manager_unfulfilled_push_requests", + "Number of unfulfilled push requests for objects.", "requests"); + +static Gauge ObjectManagerProfileEvents("object_manager_num_buffered_profile_events", + "Number of locally-buffered profile events.", + "events"); static Gauge TaskDependencyManagerStats("task_dependency_manager_stats", "Stat the metric values of task dependency.",