wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-11 04:59:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

[xray] Log warnings for asio handlers that take too long (#2601)

Browse Source 
* Add fatal check for heartbeat drift

* Log warning messages for handlers that take too long

* Add debug labels to all ClientConnections
This commit is contained in:
Stephanie Wang
2018-08-09 14:39:23 -07:00
committed by Philipp Moritz
parent d49b4bef0a
commit 2de9bfc7e3
9 changed files with 72 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/state/ray_config.h
+16
View File
@@ -12,12 +12,18 @@ class RayConfig {
int64_t ray_protocol_version() const { return ray_protocol_version_; }
uint64_t handler_warning_timeout_ms() const {
return handler_warning_timeout_ms_;
}
int64_t heartbeat_timeout_milliseconds() const {
return heartbeat_timeout_milliseconds_;
}
int64_t num_heartbeats_timeout() const { return num_heartbeats_timeout_; }
uint64_t num_heartbeats_warning() const { return num_heartbeats_warning_; }
int64_t initial_reconstruction_timeout_milliseconds() const {
return initial_reconstruction_timeout_milliseconds_;
}
@@ -113,8 +119,10 @@ class RayConfig {
private:
RayConfig()
: ray_protocol_version_(0x0000000000000000),
handler_warning_timeout_ms_(100),
heartbeat_timeout_milliseconds_(100),
num_heartbeats_timeout_(100),
num_heartbeats_warning_(5),
initial_reconstruction_timeout_milliseconds_(200),
get_timeout_milliseconds_(1000),
worker_get_request_size_(10000),
@@ -154,6 +162,10 @@ class RayConfig {
/// In theory, this is used to detect Ray version mismatches.
int64_t ray_protocol_version_;
/// The duration that a single handler on the event loop can take before a
/// warning is logged that the handler is taking too long.
uint64_t handler_warning_timeout_ms_;
/// The duration between heartbeats. These are sent by the plasma manager and
/// local scheduler.
int64_t heartbeat_timeout_milliseconds_;
@@ -161,6 +173,10 @@ class RayConfig {
/// heartbeat intervals, the global scheduler or monitor process will report
/// it as dead to the db_client table.
int64_t num_heartbeats_timeout_;
/// For a raylet, if the last heartbeat was sent more than this many
/// heartbeat periods ago, then a warning will be logged that the heartbeat
/// handler is drifting.
uint64_t num_heartbeats_warning_;
/// The initial period for a task execution lease. The lease will expire this
/// many milliseconds after the first acquisition of the lease. Nodes that