Print error when actor takes too long to start, and refactor error me… (#1747)

* Print error when actor takes too long to start, and refactor error message pushing.

* Print warning every ten seconds.

* Fix linting and tests.

* Fix tests.

src/common/state/error_table.cc

@@ -2,29 +2,21 @@
 #include "redis.h"
 
 const char *error_types[] = {"object_hash_mismatch", "put_reconstruction",
-                             "worker_died"};
-const char *error_messages[] = {
-    "A nondeterministic task was reexecuted.",
-    "An object created by ray.put was evicted and could not be reconstructed. "
-    "The driver may need to be restarted.",
-    "A worker died or was killed while executing a task."};
+                             "worker_died", "actor_not_created"};
 
 void push_error(DBHandle *db_handle,
                 DBClientID driver_id,
-                int error_index,
-                size_t data_length,
-                const unsigned char *data) {
-  RAY_CHECK(error_index >= 0 && error_index < MAX_ERROR_INDEX);
+                int error_type,
+                const std::string &error_message) {
+  int64_t message_size = error_message.size();
+
   /* Allocate a struct to hold the error information. */
-  ErrorInfo *info = (ErrorInfo *) malloc(sizeof(ErrorInfo) + data_length);
+  ErrorInfo *info = (ErrorInfo *) malloc(sizeof(ErrorInfo) + message_size);
   info->driver_id = driver_id;
-  info->error_index = error_index;
-  info->data_length = data_length;
-  memcpy(info->data, data, data_length);
-  /* Generate a random key to identify this error message. */
-  RAY_CHECK(sizeof(info->error_key) >= sizeof(UniqueID));
-  UniqueID error_key = UniqueID::from_random();
-  memcpy(info->error_key, error_key.data(), sizeof(info->error_key));
+  info->error_type = error_type;
+  info->error_key = UniqueID::from_random();
+  info->size = message_size;
+  memcpy(info->error_message, error_message.data(), message_size);
 
   init_table_callback(db_handle, UniqueID::nil(), __func__,
                       new CommonCallbackData(info), NULL, NULL,

src/common/state/error_table.h

@@ -4,50 +4,47 @@
 #include "db.h"
 #include "table.h"
 
+/// Data that is needed to push an error.
 typedef struct {
+  /// The ID of the driver to push the error to.
   DBClientID driver_id;
-  unsigned char error_key[20];
-  int error_index;
-  size_t data_length;
-  unsigned char data[0];
+  /// An index into the error_types array indicating the type of the error.
+  int error_type;
+  /// The key to use for the error message in Redis.
+  UniqueID error_key;
+  /// The length of the error message.
+  int64_t size;
+  /// The error message.
+  uint8_t error_message[0];
 } ErrorInfo;
 
-/** An error_index may be used as an index into error_types and
- *  error_messages. */
+/// An error_index may be used as an index into error_types.
 typedef enum {
-  /** An object was added with a different hash from the existing
-   *  one. */
+  /// An object was added with a different hash from the existing one.
   OBJECT_HASH_MISMATCH_ERROR_INDEX = 0,
-  /** An object that was created through a ray.put is lost. */
+  /// An object that was created through a ray.put is lost.
   PUT_RECONSTRUCTION_ERROR_INDEX,
-  /** A worker died or was killed while executing a task. */
+  /// A worker died or was killed while executing a task.
   WORKER_DIED_ERROR_INDEX,
-  /** The total number of error types. */
+  /// An actor hasn't been created for a while.
+  ACTOR_NOT_CREATED_ERROR_INDEX,
+  /// The total number of error types.
   MAX_ERROR_INDEX
 } error_index;
 
-/** Information about the error to be displayed to the user. */
 extern const char *error_types[];
-extern const char *error_messages[];
 
-/**
- * Push an error to the given Python driver.
- *
- * @param db_handle Database handle.
- * @param driver_id The ID of the Python driver to push the error
- *        to.
- * @param error_index The error information at this index in
- *        error_types and error_messages will be included in the
- *        error pushed to the driver.
- * @param data_length The length of the custom data to be included
- *        in the error.
- * @param data The custom data to be included in the error.
- * @return Void.
- */
+/// Push an error to the given Python driver.
+///
+/// \param db_handle Database handle.
+/// \param driver_id The ID of the Python driver to push the error to.
+/// \param error_type An index specifying the type of the error. This should
+/// be a value from the error_index enum.
+/// \param error_message The error message to print.
+/// \return Void.
 void push_error(DBHandle *db_handle,
                 DBClientID driver_id,
-                int error_index,
-                size_t data_length,
-                const unsigned char *data);
+                int error_type,
+                const std::string &error_message);
 
 #endif

src/common/state/ray_config.h

@@ -82,6 +82,10 @@ class RayConfig {
 
   int64_t max_tasks_to_spillback() const { return max_tasks_to_spillback_; }
 
+  int64_t actor_creation_num_spillbacks_warning() const {
+    return actor_creation_num_spillbacks_warning_;
+  }
+
  private:
   RayConfig()
       : ray_protocol_version_(0x0000000000000000),
@@ -108,7 +112,8 @@ class RayConfig {
         redis_db_connect_wait_milliseconds_(100),
         plasma_default_release_delay_(64),
         L3_cache_size_bytes_(100000000),
-        max_tasks_to_spillback_(10) {}
+        max_tasks_to_spillback_(10),
+        actor_creation_num_spillbacks_warning_(100) {}
 
   ~RayConfig() {}
 
@@ -185,6 +190,12 @@ class RayConfig {
 
   /// Constants for the spillback scheduling policy.
   int64_t max_tasks_to_spillback_;
+
+  /// Every time an actor creation task has been spilled back a number of times
+  /// that is a multiple of this quantity, a warning will be pushed to the
+  /// corresponding driver. Since spillback currently occurs on a 100ms timer,
+  /// a value of 100 corresponds to a warning every 10 seconds.
+  int64_t actor_creation_num_spillbacks_warning_;
 };
 
 #endif  // RAY_CONFIG_H

src/common/state/redis.cc

@@ -1665,7 +1665,7 @@ void redis_push_error_hmset_callback(redisAsyncContext *c,
   int status = redisAsyncCommand(
       db->context, redis_push_error_rpush_callback,
       (void *) callback_data->timer_id, "RPUSH ErrorKeys Error:%b:%b",
-      info->driver_id.data(), sizeof(info->driver_id), info->error_key,
+      info->driver_id.data(), sizeof(info->driver_id), info->error_key.data(),
       sizeof(info->error_key));
   if ((status == REDIS_ERR) || db->subscribe_context->err) {
     LOG_REDIS_DEBUG(db->subscribe_context, "error in redis_push_error rpush");
@@ -1675,18 +1675,17 @@ void redis_push_error_hmset_callback(redisAsyncContext *c,
 void redis_push_error(TableCallbackData *callback_data) {
   DBHandle *db = callback_data->db_handle;
   ErrorInfo *info = (ErrorInfo *) callback_data->data->Get();
-  RAY_CHECK(info->error_index < MAX_ERROR_INDEX && info->error_index >= 0);
-  /* Look up the error type. */
-  const char *error_type = error_types[info->error_index];
-  const char *error_message = error_messages[info->error_index];
+  RAY_CHECK(info->error_type < MAX_ERROR_INDEX && info->error_type >= 0);
+  /// Look up the error type.
+  const char *error_type = error_types[info->error_type];
 
   /* Set the error information. */
   int status = redisAsyncCommand(
       db->context, redis_push_error_hmset_callback,
       (void *) callback_data->timer_id,
-      "HMSET Error:%b:%b type %s message %s data %b", info->driver_id.data(),
-      sizeof(info->driver_id), info->error_key, sizeof(info->error_key),
-      error_type, error_message, info->data, info->data_length);
+      "HMSET Error:%b:%b type %s message %b data %b", info->driver_id.data(),
+      sizeof(info->driver_id), info->error_key.data(), sizeof(info->error_key),
+      error_type, info->error_message, info->size, "None", strlen("None"));
   if ((status == REDIS_ERR) || db->subscribe_context->err) {
     LOG_REDIS_DEBUG(db->subscribe_context, "error in redis_push_error hmset");
   }