Add timeout mechanism to Push function instead of retries (#2148)

Use timer instead of retries in Push when objects are not local.

src/common/state/ray_config.h

@@ -92,16 +92,16 @@ class RayConfig {
     return object_manager_pull_timeout_ms_;
   }
 
+  int object_manager_push_timeout_ms() const {
+    return object_manager_push_timeout_ms_;
+  }
+
   int object_manager_max_sends() const { return object_manager_max_sends_; }
 
   int object_manager_max_receives() const {
     return object_manager_max_receives_;
   }
 
-  int object_manager_max_push_retries() const {
-    return object_manager_max_push_retries_;
-  }
-
   uint64_t object_manager_default_chunk_size() const {
     return object_manager_default_chunk_size_;
   }
@@ -139,9 +139,9 @@ class RayConfig {
         // be addressed. This timeout is often on the critical path for object
         // transfers.
         object_manager_pull_timeout_ms_(20),
+        object_manager_push_timeout_ms_(10000),
         object_manager_max_sends_(2),
         object_manager_max_receives_(2),
-        object_manager_max_push_retries_(1000),
         object_manager_default_chunk_size_(100000000) {}
 
   ~RayConfig() {}
@@ -235,15 +235,18 @@ class RayConfig {
   /// ObjectManager.
   int object_manager_pull_timeout_ms_;
 
+  /// Timeout, in milliseconds, to wait until the Push request fails.
+  /// Special value:
+  /// Negative: waiting infinitely.
+  /// 0: giving up retrying immediately.
+  int object_manager_push_timeout_ms_;
+
   /// Maximum number of concurrent sends allowed by the object manager.
   int object_manager_max_sends_;
 
   /// Maximum number of concurrent receives allowed by the object manager.
   int object_manager_max_receives_;
 
-  /// Maximum push retries allowed by the object manager.
-  int object_manager_max_push_retries_;
-
   /// Default chunk size for multi-chunk transfers to use in the object manager.
   /// In the object manager, no single thread is permitted to transfer more
   /// data than what is specified by the chunk size.