wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-16 11:27:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Push errors to all drivers when node is marked dead. (#2808)

Browse Source 
* Push errors to all drivers when node is marked dead.

* Fix
This commit is contained in:
Robert Nishihara
2018-09-02 20:04:58 -07:00
committed by Philipp Moritz
parent c71bbbc3af
commit 0ac855e061
4 changed files with 63 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/raylet/monitor.cc
+14
View File
@@ -1,6 +1,7 @@
#include "ray/raylet/monitor.h"
#include "ray/status.h"
#include "ray/util/util.h"
namespace ray {
@@ -43,6 +44,19 @@ void Monitor::Tick() {
if (dead_clients_.count(it->first) == 0) {
RAY_LOG(WARNING) << "Client timed out: " << it->first;
RAY_CHECK_OK(gcs_client_.client_table().MarkDisconnected(it->first));
// Broadcast a warning to all of the drivers indicating that the node
// has been marked as dead.
// TODO(rkn): Define this constant somewhere else.
std::string type = "node_removed";
std::ostringstream error_message;
error_message << "The node with client ID " << it->first << " has been marked "
<< "dead because the monitor has missed too many heartbeats "
<< "from it.";
// We use the nil JobID to broadcast the message to all drivers.
RAY_CHECK_OK(gcs_client_.error_table().PushErrorToDriver(
JobID::nil(), type, error_message.str(), current_time_ms()));
dead_clients_.insert(it->first);
}
it = heartbeats_.erase(it);