Implement a first pass at actors in the API. (#242)

* Implement actor field for tasks

* Implement actor management in local scheduler.

* initial python frontend for actors

* import actors on worker

* IPython code completion and tests

* prepare creating actors through local schedulers

* add actor id to PyTask

* submit actor calls to local scheduler

* starting to integrate

* simple fix

* Fixes from rebasing.

* more work on python actors

* Improve local scheduler actor handlers.

* Pass actor ID to local scheduler when connecting a client.

* first working version of actors

* fixing actors

* fix creating two copies of the same actor

* fix actors

* remove sleep

* get rid of export synchronization

* update

* insert actor methods into the queue in the right order

* remove print statements

* make it compile again after rebase

* Minor updates.

* fix python actor ids

* Pass actor_id to start_worker.

* add test

* Minor changes.

* Update actor tests.

* Temporary plan for import counter.

* Temporarily fix import counters.

* Fix some tests.

* Fixes.

* Make actor creation non-blocking.

* Fix test?

* Fix actors on Python 2.

* fix rare case.

* Fix python 2 test.

* More tests.

* Small fixes.

* Linting.

* Revert tensorflow version to 0.12.0 temporarily.

* Small fix.

* Enhance inheritance test.

src/common/state/actor_notification_table.c

@@ -0,0 +1,16 @@
+#include "actor_notification_table.h"
+#include "redis.h"
+
+void actor_notification_table_subscribe(
+    db_handle *db_handle,
+    actor_notification_table_subscribe_callback subscribe_callback,
+    void *subscribe_context,
+    retry_info *retry) {
+  actor_notification_table_subscribe_data *sub_data =
+      malloc(sizeof(actor_notification_table_subscribe_data));
+  sub_data->subscribe_callback = subscribe_callback;
+  sub_data->subscribe_context = subscribe_context;
+
+  init_table_callback(db_handle, NIL_ID, __func__, sub_data, retry, NULL,
+                      redis_actor_notification_table_subscribe, NULL);
+}

src/common/state/actor_notification_table.h

@@ -0,0 +1,47 @@
+#ifndef ACTOR_NOTIFICATION_TABLE_H
+#define ACTOR_NOTIFICATION_TABLE_H
+
+#include "task.h"
+#include "db.h"
+#include "table.h"
+
+typedef struct {
+  /** The ID of the actor. */
+  actor_id actor_id;
+  /** The ID of the local scheduler that is responsible for the actor. */
+  db_client_id local_scheduler_id;
+} actor_info;
+
+/*
+ *  ==== Subscribing to the actor notification table ====
+ */
+
+/* Callback for subscribing to the local scheduler table. */
+typedef void (*actor_notification_table_subscribe_callback)(actor_info info,
+                                                            void *user_context);
+
+/**
+ * Register a callback to process actor notification events.
+ *
+ * @param db_handle Database handle.
+ * @param subscribe_callback Callback that will be called when the local
+ *        scheduler event happens.
+ * @param subscribe_context Context that will be passed into the
+ *        subscribe_callback.
+ * @param retry Information about retrying the request to the database.
+ * @return Void.
+ */
+void actor_notification_table_subscribe(
+    db_handle *db_handle,
+    actor_notification_table_subscribe_callback subscribe_callback,
+    void *subscribe_context,
+    retry_info *retry);
+
+/* Data that is needed to register local scheduler table subscribe callbacks
+ * with the state database. */
+typedef struct {
+  actor_notification_table_subscribe_callback subscribe_callback;
+  void *subscribe_context;
+} actor_notification_table_subscribe_data;
+
+#endif /* ACTOR_NOTIFICATION_TABLE_H */

src/common/state/local_scheduler_table.h

@@ -70,7 +70,7 @@ void local_scheduler_table_send_info(db_handle *db_handle,
                                      local_scheduler_info *info,
                                      retry_info *retry);
 
-/* Data that is needed to publish local scheduer heartbeats to the local
+/* Data that is needed to publish local scheduler heartbeats to the local
  * scheduler table. */
 typedef struct {
   local_scheduler_info info;

src/common/state/redis.c

@@ -12,6 +12,7 @@
 #include "common.h"
 #include "db.h"
 #include "db_client_table.h"
+#include "actor_notification_table.h"
 #include "local_scheduler_table.h"
 #include "object_table.h"
 #include "object_info.h"
@@ -1063,7 +1064,7 @@ void redis_local_scheduler_table_subscribe_callback(redisAsyncContext *c,
   CHECK(reply->type == REDIS_REPLY_ARRAY);
   CHECK(reply->elements == 3);
   redisReply *message_type = reply->element[0];
-  LOG_DEBUG("Local scheduer table subscribe callback, message %s",
+  LOG_DEBUG("Local scheduler table subscribe callback, message %s",
             message_type->str);
 
   if (strcmp(message_type->str, "message") == 0) {
@@ -1130,6 +1131,57 @@ void redis_local_scheduler_table_send_info(table_callback_data *callback_data) {
   }
 }
 
+void redis_actor_notification_table_subscribe_callback(redisAsyncContext *c,
+                                                       void *r,
+                                                       void *privdata) {
+  REDIS_CALLBACK_HEADER(db, callback_data, r);
+
+  redisReply *reply = r;
+  CHECK(reply->type == REDIS_REPLY_ARRAY);
+  CHECK(reply->elements == 3);
+  redisReply *message_type = reply->element[0];
+  LOG_DEBUG("Local scheduler table subscribe callback, message %s",
+            message_type->str);
+
+  if (strcmp(message_type->str, "message") == 0) {
+    /* Handle an actor notification message. Parse the payload and call the
+     * subscribe callback. */
+    redisReply *payload = reply->element[2];
+    actor_notification_table_subscribe_data *data = callback_data->data;
+    actor_info info;
+    /* The payload should be the concatenation of these two structs. */
+    CHECK(sizeof(info.actor_id) + sizeof(info.local_scheduler_id) ==
+          payload->len);
+    memcpy(&info.actor_id, payload->str, sizeof(info.actor_id));
+    memcpy(&info.local_scheduler_id, payload->str + sizeof(info.actor_id),
+           sizeof(info.local_scheduler_id));
+    if (data->subscribe_callback) {
+      data->subscribe_callback(info, data->subscribe_context);
+    }
+  } else if (strcmp(message_type->str, "subscribe") == 0) {
+    /* The reply for the initial SUBSCRIBE command. */
+    CHECK(callback_data->done_callback == NULL);
+    /* If the initial SUBSCRIBE was successful, clean up the timer, but don't
+     * destroy the callback data. */
+    event_loop_remove_timer(db->loop, callback_data->timer_id);
+
+  } else {
+    LOG_FATAL("Unexpected reply type from actor notification subscribe.");
+  }
+}
+
+void redis_actor_notification_table_subscribe(
+    table_callback_data *callback_data) {
+  db_handle *db = callback_data->db_handle;
+  int status = redisAsyncCommand(
+      db->sub_context, redis_actor_notification_table_subscribe_callback,
+      (void *) callback_data->timer_id, "SUBSCRIBE actor_notifications");
+  if ((status == REDIS_ERR) || db->sub_context->err) {
+    LOG_REDIS_DEBUG(db->sub_context,
+                    "error in redis_actor_notification_table_subscribe");
+  }
+}
+
 void redis_object_info_subscribe_callback(redisAsyncContext *c,
                                           void *r,
                                           void *privdata) {

src/common/state/redis.h

@@ -245,6 +245,16 @@ void redis_local_scheduler_table_subscribe(table_callback_data *callback_data);
  */
 void redis_local_scheduler_table_send_info(table_callback_data *callback_data);
 
+/**
+ * Subscribe to updates about newly created actors.
+ *
+ * @param callback_data Data structure containing redis connection and timeout
+ *        information.
+ * @return Void.
+ */
+void redis_actor_notification_table_subscribe(
+    table_callback_data *callback_data);
+
 void redis_object_info_subscribe(table_callback_data *callback_data);
 
 #endif /* REDIS_H */