diff --git a/src/ray/object_manager/plasma/store.cc b/src/ray/object_manager/plasma/store.cc
index 9bae68b3a..d12a8096a 100644
--- a/src/ray/object_manager/plasma/store.cc
+++ b/src/ray/object_manager/plasma/store.cc
@@ -562,6 +562,10 @@ void PlasmaStore::EraseFromObjectTable(const ObjectID &object_id) {
   if (object->device_num == 0) {
     PlasmaAllocator::Free(object->pointer, buff_size);
   }
+  if (object->ref_count > 0) {
+    // A client was using this object.
+    num_bytes_in_use_ -= object->data_size + object->metadata_size;
+  }
   store_info_.objects.erase(object_id);
 }