From b73d4831d4eca9add44f359bb38880f29d365c33 Mon Sep 17 00:00:00 2001
From: Eric Liang <ekhliang@gmail.com>
Date: Sat, 12 Dec 2020 12:02:13 -0800
Subject: [PATCH]  Add grace period before warning of resource deadlock

---
 src/ray/raylet/node_manager.cc | 13 ++++++++-----
 src/ray/raylet/node_manager.h  |  5 +++--
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc
index 1e1f06104..e2e39fe30 100644
--- a/src/ray/raylet/node_manager.cc
+++ b/src/ray/raylet/node_manager.cc
@@ -648,7 +648,7 @@ void NodeManager::WarnResourceDeadlock() {
       continue;
     }
     // Progress is being made, don't warn.
-    resource_deadlock_warned_ = false;
+    resource_deadlock_warned_ = 0;
     return;
   }
 
@@ -675,13 +675,17 @@ void NodeManager::WarnResourceDeadlock() {
   }
 
   // Push an warning to the driver that a task is blocked trying to acquire resources.
-  if (any_pending) {
+  // To avoid spurious triggers, only take action starting with the second time.
+  // case resource_deadlock_warned_:  0 => first time, don't do anything yet
+  // case resource_deadlock_warned_:  1 => second time, print a warning
+  // case resource_deadlock_warned_: >1 => global gc but don't print any warnings
+  if (any_pending && resource_deadlock_warned_++ > 0) {
     // Actor references may be caught in cycles, preventing them from being deleted.
     // Trigger global GC to hopefully free up resource slots.
     TriggerGlobalGC();
 
     // Suppress duplicates warning messages.
-    if (resource_deadlock_warned_) {
+    if (resource_deadlock_warned_ > 2) {
       return;
     }
 
@@ -702,7 +706,6 @@ void NodeManager::WarnResourceDeadlock() {
         "resource_deadlock", error_message.str(), current_time_ms(),
         exemplar.GetTaskSpecification().JobId());
     RAY_CHECK_OK(gcs_client_->Errors().AsyncReportJobError(error_data_ptr, nullptr));
-    resource_deadlock_warned_ = true;
   }
 }
 
@@ -3226,7 +3229,7 @@ void NodeManager::HandleGlobalGC(const rpc::GlobalGCRequest &request,
 
 void NodeManager::TriggerGlobalGC() {
   RAY_LOG(INFO) << "Broadcasting Python GC request to all raylets since the cluster "
-                << "is low on object store memory. This removes Ray object refs "
+                << "is low on resources. This removes Ray actor and object refs "
                 << "that are stuck in Python reference cycles.";
   should_global_gc_ = true;
   // We won't see our own request, so trigger local GC in the next heartbeat.
diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h
index 7a31da54b..9d8696a88 100644
--- a/src/ray/raylet/node_manager.h
+++ b/src/ray/raylet/node_manager.h
@@ -695,8 +695,9 @@ class NodeManager : public rpc::NodeManagerServiceHandler {
   bool fair_queueing_enabled_;
   /// Whether to enable pinning for plasma objects.
   bool object_pinning_enabled_;
-  /// Whether we have printed out a resource deadlock warning.
-  bool resource_deadlock_warned_ = false;
+  /// Incremented each time we encounter a potential resource deadlock condition.
+  /// This is reset to zero when the condition is cleared.
+  int resource_deadlock_warned_ = 0;
   /// Whether we have recorded any metrics yet.
   bool recorded_metrics_ = false;
   /// The path to the ray temp dir.