Warn the user when a nondeterministic task is detected. (#339)

* WARN instead of FATAL for object hash mismatches, push error to driver

* Document the callback signature for object_table_add/remove

* Error table

* Wait for all errors in python test

* Fix doc

* Fix state test

src/common/state/error_table.cc

@@ -0,0 +1,23 @@
+#include "error_table.h"
+#include "redis.h"
+
+void push_error(DBHandle *db_handle,
+                DBClientID driver_id,
+                int error_index,
+                size_t data_length,
+                unsigned char *data) {
+  CHECK(error_index >= 0 && error_index < MAX_ERROR_INDEX);
+  /* Allocate a struct to hold the error information. */
+  ErrorInfo *info = (ErrorInfo *) malloc(sizeof(ErrorInfo) + data_length);
+  info->driver_id = driver_id;
+  info->error_index = error_index;
+  info->data_length = data_length;
+  memcpy(info->data, data, data_length);
+  /* Generate a random key to identify this error message. */
+  CHECK(sizeof(info->error_key) >= UNIQUE_ID_SIZE);
+  UniqueID error_key = globally_unique_id();
+  memcpy(info->error_key, error_key.id, sizeof(info->error_key));
+
+  init_table_callback(db_handle, NIL_ID, __func__, info, NULL, NULL,
+                      redis_push_error, NULL);
+}

src/common/state/error_table.h

@@ -0,0 +1,50 @@
+#ifndef ERROR_TABLE_H
+#define ERROR_TABLE_H
+
+#include "db.h"
+#include "table.h"
+
+typedef struct {
+  DBClientID driver_id;
+  unsigned char error_key[20];
+  int error_index;
+  size_t data_length;
+  unsigned char data[0];
+} ErrorInfo;
+
+/** An error_index may be used as an index into error_types and
+ *  error_messages. */
+typedef enum {
+  /** An object was added with a different hash from the existing
+   *  one. */
+  OBJECT_HASH_MISMATCH_ERROR_INDEX = 0,
+  /** The total number of error types. */
+  MAX_ERROR_INDEX
+} error_index;
+
+/** Information about the error to be displayed to the user. */
+static const char *error_types[] = {"object_hash_mismatch"};
+static const char *error_messages[] = {
+    "A nondeterministic task was reexecuted."};
+
+/**
+ * Push an error to the given Python driver.
+ *
+ * @param db_handle Database handle.
+ * @param driver_id The ID of the Python driver to push the error
+ *        to.
+ * @param error_index The error information at this index in
+ *        error_types and error_messages will be included in the
+ *        error pushed to the driver.
+ * @param data_length The length of the custom data to be included
+ *        in the error.
+ * @param data The custom data to be included in the error.
+ * @return Void.
+ */
+void push_error(DBHandle *db_handle,
+                DBClientID driver_id,
+                int error_index,
+                size_t data_length,
+                unsigned char *data);
+
+#endif

src/common/state/object_table.h

@@ -49,8 +49,18 @@ void object_table_lookup(DBHandle *db_handle,
  *  ==== Add object call and callback ====
  */
 
-/* Callback called when the object add/remove operation completes. */
+/**
+ * Callback called when the object add/remove operation completes.
+ *
+ * @param object_id The ID of the object that was added or removed.
+ * @param success Whether the operation was successful or not. If this is false
+ *        and the operation was an addition, the object was added, but there
+ *        was a hash mismatch.
+ * @param user_context The user context that was passed into the add/remove
+ *        call.
+ */
 typedef void (*object_table_done_callback)(ObjectID object_id,
+                                           bool success,
                                            void *user_context);
 
 /**

src/common/state/redis.cc

@@ -22,6 +22,7 @@ extern "C" {
 #include "object_info.h"
 #include "task.h"
 #include "task_table.h"
+#include "error_table.h"
 #include "event_loop.h"
 #include "redis.h"
 #include "io.h"
@@ -217,21 +218,23 @@ void redis_object_table_add_callback(redisAsyncContext *c,
 
   /* Do some minimal checking. */
   redisReply *reply = (redisReply *) r;
-  if (strcmp(reply->str, "hash mismatch") == 0) {
+  bool success = (strcmp(reply->str, "hash mismatch") != 0);
+  if (!success) {
     /* If our object hash doesn't match the one recorded in the table, report
      * the error back to the user and exit immediately. */
-    LOG_FATAL(
+    LOG_WARN(
         "Found objects with different value but same object ID, most likely "
         "because a nondeterministic task was executed twice, either for "
         "reconstruction or for speculation.");
+  } else {
+    CHECK(reply->type != REDIS_REPLY_ERROR);
+    CHECK(strcmp(reply->str, "OK") == 0);
   }
-  CHECK(reply->type != REDIS_REPLY_ERROR);
-  CHECK(strcmp(reply->str, "OK") == 0);
   /* Call the done callback if there is one. */
   if (callback_data->done_callback != NULL) {
     object_table_done_callback done_callback =
         (object_table_done_callback) callback_data->done_callback;
-    done_callback(callback_data->id, callback_data->user_context);
+    done_callback(callback_data->id, success, callback_data->user_context);
   }
   /* Clean up the timer and callback. */
   destroy_timer_callback(db->loop, callback_data);
@@ -274,7 +277,7 @@ void redis_object_table_remove_callback(redisAsyncContext *c,
   if (callback_data->done_callback != NULL) {
     object_table_done_callback done_callback =
         (object_table_done_callback) callback_data->done_callback;
-    done_callback(callback_data->id, callback_data->user_context);
+    done_callback(callback_data->id, true, callback_data->user_context);
   }
   /* Clean up the timer and callback. */
   destroy_timer_callback(db->loop, callback_data);
@@ -1275,6 +1278,58 @@ void redis_object_info_subscribe(TableCallbackData *callback_data) {
   }
 }
 
+void redis_push_error_rpush_callback(redisAsyncContext *c,
+                                     void *r,
+                                     void *privdata) {
+  REDIS_CALLBACK_HEADER(db, callback_data, r);
+  redisReply *reply = (redisReply *) r;
+  /* The reply should be the length of the errors list after our RPUSH. */
+  CHECK(reply->type == REDIS_REPLY_INTEGER);
+  destroy_timer_callback(db->loop, callback_data);
+}
+
+void redis_push_error_hmset_callback(redisAsyncContext *c,
+                                     void *r,
+                                     void *privdata) {
+  REDIS_CALLBACK_HEADER(db, callback_data, r);
+  redisReply *reply = (redisReply *) r;
+
+  /* Make sure we were able to add the error information. */
+  CHECK(reply->type != REDIS_REPLY_ERROR);
+  CHECK(strcmp(reply->str, "OK") == 0);
+
+  /* Add the error to this driver's list of errors. */
+  ErrorInfo *info = (ErrorInfo *) callback_data->data;
+  int status = redisAsyncCommand(db->context, redis_push_error_rpush_callback,
+                                 (void *) callback_data->timer_id,
+                                 "RPUSH ErrorKeys Error:%b:%b",
+                                 info->driver_id.id, sizeof(info->driver_id.id),
+                                 info->error_key, sizeof(info->error_key));
+  if ((status == REDIS_ERR) || db->sub_context->err) {
+    LOG_REDIS_DEBUG(db->sub_context, "error in redis_push_error rpush");
+  }
+}
+
+void redis_push_error(TableCallbackData *callback_data) {
+  DBHandle *db = callback_data->db_handle;
+  ErrorInfo *info = (ErrorInfo *) callback_data->data;
+  CHECK(info->error_index < MAX_ERROR_INDEX && info->error_index >= 0);
+  /* Look up the error type. */
+  const char *error_type = error_types[info->error_index];
+  const char *error_message = error_messages[info->error_index];
+
+  /* Set the error information. */
+  int status = redisAsyncCommand(
+      db->context, redis_push_error_hmset_callback,
+      (void *) callback_data->timer_id,
+      "HMSET Error:%b:%b type %s message %s data %b", info->driver_id.id,
+      sizeof(info->driver_id.id), info->error_key, sizeof(info->error_key),
+      error_type, error_message, info->data, info->data_length);
+  if ((status == REDIS_ERR) || db->sub_context->err) {
+    LOG_REDIS_DEBUG(db->sub_context, "error in redis_push_error hmset");
+  }
+}
+
 DBClientID get_db_client_id(DBHandle *db) {
   CHECK(db != NULL);
   return db->client;

src/common/state/redis.h

@@ -264,4 +264,6 @@ void redis_actor_notification_table_subscribe(TableCallbackData *callback_data);
 
 void redis_object_info_subscribe(TableCallbackData *callback_data);
 
+void redis_push_error(TableCallbackData *callback_data);
+
 #endif /* REDIS_H */