From da89cb19eb5a1e4a7953a32259acc95a949fd759 Mon Sep 17 00:00:00 2001 From: fangfengbin <869218239a@zju.edu.cn> Date: Mon, 19 Oct 2020 10:23:33 +0800 Subject: [PATCH] [GCS]Fix node info idempotent bug (#11423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [GCS]Fix node info idempotent bug * Fix review comment Co-authored-by: 灵洵 --- .../gcs/gcs_client/service_based_accessor.cc | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/ray/gcs/gcs_client/service_based_accessor.cc b/src/ray/gcs/gcs_client/service_based_accessor.cc index 9133e564b..a931acfa3 100644 --- a/src/ray/gcs/gcs_client/service_based_accessor.cc +++ b/src/ray/gcs/gcs_client/service_based_accessor.cc @@ -753,12 +753,21 @@ void ServiceBasedNodeInfoAccessor::HandleNotification(const GcsNodeInfo &node_in // was alive and is now dead or resources have been updated. bool was_alive = (entry->second.state() == GcsNodeInfo::ALIVE); is_notif_new = was_alive && !is_alive; + // Once a node with a given ID has been removed, it should never be added - // again. If the entry was in the cache and the node was deleted, check + // again. If the entry was in the cache and the node was deleted, we should check // that this new notification is not an insertion. - if (!was_alive) { - RAY_CHECK(!is_alive) - << "Notification for addition of a node that was already removed:" << node_id; + // However, when a new node(node-B) registers with GCS, it subscribes to all node + // information. It will subscribe to redis and then get all node information from GCS + // through RPC. If node-A fails after GCS replies to node-B, GCS will send another + // message(node-A is dead) to node-B through redis publish. Because RPC and redis + // subscribe are two different sessions, node-B may process node-A dead message first + // and then node-A alive message. So we use `RAY_LOG` instead of `RAY_CHECK ` as a + // workaround. + if (!was_alive && is_alive) { + RAY_LOG(INFO) << "Notification for addition of a node that was already removed:" + << node_id; + return; } }