Implement a first pass at actors in the API. (#242)

* Implement actor field for tasks

* Implement actor management in local scheduler.

* initial python frontend for actors

* import actors on worker

* IPython code completion and tests

* prepare creating actors through local schedulers

* add actor id to PyTask

* submit actor calls to local scheduler

* starting to integrate

* simple fix

* Fixes from rebasing.

* more work on python actors

* Improve local scheduler actor handlers.

* Pass actor ID to local scheduler when connecting a client.

* first working version of actors

* fixing actors

* fix creating two copies of the same actor

* fix actors

* remove sleep

* get rid of export synchronization

* update

* insert actor methods into the queue in the right order

* remove print statements

* make it compile again after rebase

* Minor updates.

* fix python actor ids

* Pass actor_id to start_worker.

* add test

* Minor changes.

* Update actor tests.

* Temporary plan for import counter.

* Temporarily fix import counters.

* Fix some tests.

* Fixes.

* Make actor creation non-blocking.

* Fix test?

* Fix actors on Python 2.

* fix rare case.

* Fix python 2 test.

* More tests.

* Small fixes.

* Linting.

* Revert tensorflow version to 0.12.0 temporarily.

* Small fix.

* Enhance inheritance test.

src/common/CMakeLists.txt

@@ -25,6 +25,7 @@ add_library(common STATIC
   state/object_table.c
   state/task_table.c
   state/db_client_table.c
+  state/actor_notification_table.c
   state/local_scheduler_table.c
   thirdparty/ae/ae.c
   thirdparty/sha256.c)

src/common/lib/python/common_extension.c

@@ -263,7 +263,13 @@ PyTypeObject PyObjectIDType = {
 /* Define the PyTask class. */
 
 static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
+  /* ID of the driver that this task originates from. */
   unique_id driver_id;
+  /* ID of the actor this task should run on. */
+  unique_id actor_id = NIL_ACTOR_ID;
+  /* How many tasks have been launched on the actor so far? */
+  int actor_counter = 0;
+  /* ID of the function this task executes. */
   function_id function_id;
   /* Arguments of the task (can be PyObjectIDs or Python values). */
   PyObject *arguments;
@@ -277,10 +283,11 @@ static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
   int parent_counter;
   /* Resource vector of the required resources to execute this task. */
   PyObject *resource_vector = NULL;
-  if (!PyArg_ParseTuple(args, "O&O&OiO&i|O", &PyObjectToUniqueID, &driver_id,
+  if (!PyArg_ParseTuple(args, "O&O&OiO&i|O&iO", &PyObjectToUniqueID, &driver_id,
                         &PyObjectToUniqueID, &function_id, &arguments,
                         &num_returns, &PyObjectToUniqueID, &parent_task_id,
-                        &parent_counter, &resource_vector)) {
+                        &parent_counter, &PyObjectToUniqueID, &actor_id,
+                        &actor_counter, &resource_vector)) {
     return -1;
   }
   Py_ssize_t size = PyList_Size(arguments);
@@ -299,9 +306,9 @@ static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
   }
   /* Construct the task specification. */
   int val_repr_index = 0;
-  self->spec = start_construct_task_spec(driver_id, parent_task_id,
-                                         parent_counter, function_id, size,
-                                         num_returns, value_data_bytes);
+  self->spec = start_construct_task_spec(
+      driver_id, parent_task_id, parent_counter, actor_id, actor_counter,
+      function_id, size, num_returns, value_data_bytes);
   /* Add the task arguments. */
   for (Py_ssize_t i = 0; i < size; ++i) {
     PyObject *arg = PyList_GetItem(arguments, i);
@@ -350,6 +357,11 @@ static PyObject *PyTask_function_id(PyObject *self) {
   return PyObjectID_make(function_id);
 }
 
+static PyObject *PyTask_actor_id(PyObject *self) {
+  actor_id actor_id = task_spec_actor_id(((PyTask *) self)->spec);
+  return PyObjectID_make(actor_id);
+}
+
 static PyObject *PyTask_driver_id(PyObject *self) {
   unique_id driver_id = task_spec_driver_id(((PyTask *) self)->spec);
   return PyObjectID_make(driver_id);
@@ -407,6 +419,8 @@ static PyObject *PyTask_returns(PyObject *self) {
 static PyMethodDef PyTask_methods[] = {
     {"function_id", (PyCFunction) PyTask_function_id, METH_NOARGS,
      "Return the function ID for this task."},
+    {"actor_id", (PyCFunction) PyTask_actor_id, METH_NOARGS,
+     "Return the actor ID for this task."},
     {"driver_id", (PyCFunction) PyTask_driver_id, METH_NOARGS,
      "Return the driver ID for this task."},
     {"task_id", (PyCFunction) PyTask_task_id, METH_NOARGS,

src/common/state/actor_notification_table.c

@@ -0,0 +1,16 @@
+#include "actor_notification_table.h"
+#include "redis.h"
+
+void actor_notification_table_subscribe(
+    db_handle *db_handle,
+    actor_notification_table_subscribe_callback subscribe_callback,
+    void *subscribe_context,
+    retry_info *retry) {
+  actor_notification_table_subscribe_data *sub_data =
+      malloc(sizeof(actor_notification_table_subscribe_data));
+  sub_data->subscribe_callback = subscribe_callback;
+  sub_data->subscribe_context = subscribe_context;
+
+  init_table_callback(db_handle, NIL_ID, __func__, sub_data, retry, NULL,
+                      redis_actor_notification_table_subscribe, NULL);
+}

src/common/state/actor_notification_table.h

@@ -0,0 +1,47 @@
+#ifndef ACTOR_NOTIFICATION_TABLE_H
+#define ACTOR_NOTIFICATION_TABLE_H
+
+#include "task.h"
+#include "db.h"
+#include "table.h"
+
+typedef struct {
+  /** The ID of the actor. */
+  actor_id actor_id;
+  /** The ID of the local scheduler that is responsible for the actor. */
+  db_client_id local_scheduler_id;
+} actor_info;
+
+/*
+ *  ==== Subscribing to the actor notification table ====
+ */
+
+/* Callback for subscribing to the local scheduler table. */
+typedef void (*actor_notification_table_subscribe_callback)(actor_info info,
+                                                            void *user_context);
+
+/**
+ * Register a callback to process actor notification events.
+ *
+ * @param db_handle Database handle.
+ * @param subscribe_callback Callback that will be called when the local
+ *        scheduler event happens.
+ * @param subscribe_context Context that will be passed into the
+ *        subscribe_callback.
+ * @param retry Information about retrying the request to the database.
+ * @return Void.
+ */
+void actor_notification_table_subscribe(
+    db_handle *db_handle,
+    actor_notification_table_subscribe_callback subscribe_callback,
+    void *subscribe_context,
+    retry_info *retry);
+
+/* Data that is needed to register local scheduler table subscribe callbacks
+ * with the state database. */
+typedef struct {
+  actor_notification_table_subscribe_callback subscribe_callback;
+  void *subscribe_context;
+} actor_notification_table_subscribe_data;
+
+#endif /* ACTOR_NOTIFICATION_TABLE_H */

src/common/state/local_scheduler_table.h

@@ -70,7 +70,7 @@ void local_scheduler_table_send_info(db_handle *db_handle,
                                      local_scheduler_info *info,
                                      retry_info *retry);
 
-/* Data that is needed to publish local scheduer heartbeats to the local
+/* Data that is needed to publish local scheduler heartbeats to the local
  * scheduler table. */
 typedef struct {
   local_scheduler_info info;

src/common/state/redis.c

@@ -12,6 +12,7 @@
 #include "common.h"
 #include "db.h"
 #include "db_client_table.h"
+#include "actor_notification_table.h"
 #include "local_scheduler_table.h"
 #include "object_table.h"
 #include "object_info.h"
@@ -1063,7 +1064,7 @@ void redis_local_scheduler_table_subscribe_callback(redisAsyncContext *c,
   CHECK(reply->type == REDIS_REPLY_ARRAY);
   CHECK(reply->elements == 3);
   redisReply *message_type = reply->element[0];
-  LOG_DEBUG("Local scheduer table subscribe callback, message %s",
+  LOG_DEBUG("Local scheduler table subscribe callback, message %s",
             message_type->str);
 
   if (strcmp(message_type->str, "message") == 0) {
@@ -1130,6 +1131,57 @@ void redis_local_scheduler_table_send_info(table_callback_data *callback_data) {
   }
 }
 
+void redis_actor_notification_table_subscribe_callback(redisAsyncContext *c,
+                                                       void *r,
+                                                       void *privdata) {
+  REDIS_CALLBACK_HEADER(db, callback_data, r);
+
+  redisReply *reply = r;
+  CHECK(reply->type == REDIS_REPLY_ARRAY);
+  CHECK(reply->elements == 3);
+  redisReply *message_type = reply->element[0];
+  LOG_DEBUG("Local scheduler table subscribe callback, message %s",
+            message_type->str);
+
+  if (strcmp(message_type->str, "message") == 0) {
+    /* Handle an actor notification message. Parse the payload and call the
+     * subscribe callback. */
+    redisReply *payload = reply->element[2];
+    actor_notification_table_subscribe_data *data = callback_data->data;
+    actor_info info;
+    /* The payload should be the concatenation of these two structs. */
+    CHECK(sizeof(info.actor_id) + sizeof(info.local_scheduler_id) ==
+          payload->len);
+    memcpy(&info.actor_id, payload->str, sizeof(info.actor_id));
+    memcpy(&info.local_scheduler_id, payload->str + sizeof(info.actor_id),
+           sizeof(info.local_scheduler_id));
+    if (data->subscribe_callback) {
+      data->subscribe_callback(info, data->subscribe_context);
+    }
+  } else if (strcmp(message_type->str, "subscribe") == 0) {
+    /* The reply for the initial SUBSCRIBE command. */
+    CHECK(callback_data->done_callback == NULL);
+    /* If the initial SUBSCRIBE was successful, clean up the timer, but don't
+     * destroy the callback data. */
+    event_loop_remove_timer(db->loop, callback_data->timer_id);
+
+  } else {
+    LOG_FATAL("Unexpected reply type from actor notification subscribe.");
+  }
+}
+
+void redis_actor_notification_table_subscribe(
+    table_callback_data *callback_data) {
+  db_handle *db = callback_data->db_handle;
+  int status = redisAsyncCommand(
+      db->sub_context, redis_actor_notification_table_subscribe_callback,
+      (void *) callback_data->timer_id, "SUBSCRIBE actor_notifications");
+  if ((status == REDIS_ERR) || db->sub_context->err) {
+    LOG_REDIS_DEBUG(db->sub_context,
+                    "error in redis_actor_notification_table_subscribe");
+  }
+}
+
 void redis_object_info_subscribe_callback(redisAsyncContext *c,
                                           void *r,
                                           void *privdata) {

src/common/state/redis.h

@@ -245,6 +245,16 @@ void redis_local_scheduler_table_subscribe(table_callback_data *callback_data);
  */
 void redis_local_scheduler_table_send_info(table_callback_data *callback_data);
 
+/**
+ * Subscribe to updates about newly created actors.
+ *
+ * @param callback_data Data structure containing redis connection and timeout
+ *        information.
+ * @return Void.
+ */
+void redis_actor_notification_table_subscribe(
+    table_callback_data *callback_data);
+
 void redis_object_info_subscribe(table_callback_data *callback_data);
 
 #endif /* REDIS_H */

src/common/task.c

@@ -44,6 +44,11 @@ struct task_spec_impl {
   /** A count of the number of tasks submitted by the parent task before this
    *  one. */
   int64_t parent_counter;
+  /** Actor ID of the task. This is the actor that this task is executed on
+   *  or NIL_ACTOR_ID if the task is just a normal task. */
+  actor_id actor_id;
+  /** Number of tasks that have been submitted to this actor so far. */
+  int64_t actor_counter;
   /** Function ID of the task. */
   function_id function_id;
   /** Total number of arguments. */
@@ -81,6 +86,10 @@ bool task_id_is_nil(task_id id) {
   return task_ids_equal(id, NIL_TASK_ID);
 }
 
+bool actor_ids_equal(actor_id first_id, actor_id second_id) {
+  return UNIQUE_ID_EQ(first_id, second_id);
+}
+
 bool function_ids_equal(function_id first_id, function_id second_id) {
   return UNIQUE_ID_EQ(first_id, second_id);
 }
@@ -147,6 +156,8 @@ object_id task_compute_put_id(task_id task_id, int64_t put_index) {
 task_spec *start_construct_task_spec(unique_id driver_id,
                                      task_id parent_task_id,
                                      int64_t parent_counter,
+                                     actor_id actor_id,
+                                     int64_t actor_counter,
                                      function_id function_id,
                                      int64_t num_args,
                                      int64_t num_returns,
@@ -158,6 +169,8 @@ task_spec *start_construct_task_spec(unique_id driver_id,
   task->task_id = NIL_TASK_ID;
   task->parent_task_id = parent_task_id;
   task->parent_counter = parent_counter;
+  task->actor_id = actor_id;
+  task->actor_counter = actor_counter;
   task->function_id = function_id;
   task->num_args = num_args;
   task->arg_index = 0;
@@ -190,6 +203,18 @@ function_id task_function(task_spec *spec) {
   return spec->function_id;
 }
 
+actor_id task_spec_actor_id(task_spec *spec) {
+  /* Check that the task has been constructed. */
+  DCHECK(!task_ids_equal(spec->task_id, NIL_TASK_ID));
+  return spec->actor_id;
+}
+
+int64_t task_spec_actor_counter(task_spec *spec) {
+  /* Check that the task has been constructed. */
+  DCHECK(!task_ids_equal(spec->task_id, NIL_TASK_ID));
+  return spec->actor_counter;
+}
+
 unique_id task_spec_driver_id(task_spec *spec) {
   /* Check that the task has been constructed. */
   DCHECK(!task_ids_equal(spec->task_id, NIL_TASK_ID));

src/common/task.h

@@ -15,6 +15,7 @@
 #include "utstring.h"
 
 #define NIL_TASK_ID NIL_ID
+#define NIL_ACTOR_ID NIL_ID
 #define NIL_FUNCTION_ID NIL_ID
 
 typedef unique_id function_id;
@@ -23,6 +24,10 @@ typedef unique_id function_id;
  *  executes and the argument IDs or argument values. */
 typedef unique_id task_id;
 
+/** The actor ID is the ID of the actor that a task must run on. If the task is
+ *  not run on an actor, then NIL_ACTOR_ID should be used. */
+typedef unique_id actor_id;
+
 /** The task instance ID is a globally unique ID generated which identifies this
  *  particular execution of the task. */
 typedef unique_id task_iid;
@@ -55,6 +60,15 @@ bool task_ids_equal(task_id first_id, task_id second_id);
  */
 bool task_id_is_nil(task_id id);
 
+/**
+ * Compare two actor IDs.
+ *
+ * @param first_id The first actor ID to compare.
+ * @param second_id The first actor ID to compare.
+ * @return True if the actor IDs are the same and false otherwise.
+ */
+bool actor_ids_equal(actor_id first_id, actor_id second_id);
+
 /**
  * Compare two function IDs.
  *
@@ -83,6 +97,8 @@ bool function_id_is_nil(function_id id);
  * @param parent_task_id The task ID of the task that submitted this task.
  * @param parent_counter A counter indicating how many tasks were submitted by
  *        the parent task prior to this one.
+ * @param actor_id The ID of the actor this task belongs to.
+ * @param actor_counter Number of tasks that have been executed on this actor.
  * @param function_id The function ID of the function to execute in this task.
  * @param num_args The number of arguments that this task has.
  * @param num_returns The number of return values that this task has.
@@ -93,6 +109,8 @@ bool function_id_is_nil(function_id id);
 task_spec *start_construct_task_spec(unique_id driver_id,
                                      task_id parent_task_id,
                                      int64_t parent_counter,
+                                     unique_id actor_id,
+                                     int64_t actor_counter,
                                      function_id function_id,
                                      int64_t num_args,
                                      int64_t num_returns,
@@ -124,6 +142,23 @@ int64_t task_spec_size(task_spec *spec);
  */
 function_id task_function(task_spec *spec);
 
+/**
+ * Return the actor ID of the task.
+ *
+ * @param spec The task_spec in question.
+ * @return The actor ID of the actor the task is part of.
+ */
+unique_id task_spec_actor_id(task_spec *spec);
+
+/**
+ * Return the actor counter of the task. This starts at 0 and increments by 1
+ * every time a new task is submitted to run on the actor.
+ *
+ * @param spec The task_spec in question.
+ * @return The actor counter of the task.
+ */
+int64_t task_spec_actor_counter(task_spec *spec);
+
 /**
  * Return the driver ID of the task.
  *

src/common/test/task_tests.c

@@ -14,8 +14,8 @@ SUITE(task_tests);
 TEST task_test(void) {
   task_id parent_task_id = globally_unique_id();
   function_id func_id = globally_unique_id();
-  task_spec *spec =
-      start_construct_task_spec(NIL_ID, parent_task_id, 0, func_id, 4, 2, 10);
+  task_spec *spec = start_construct_task_spec(
+      NIL_ID, parent_task_id, 0, NIL_ACTOR_ID, 0, func_id, 4, 2, 10);
   ASSERT(task_num_args(spec) == 4);
   ASSERT(task_num_returns(spec) == 2);
 
@@ -52,15 +52,15 @@ TEST deterministic_ids_test(void) {
   uint8_t *arg2 = (uint8_t *) "hello world";
 
   /* Construct a first task. */
-  task_spec *spec1 =
-      start_construct_task_spec(NIL_ID, parent_task_id, 0, func_id, 2, 3, 11);
+  task_spec *spec1 = start_construct_task_spec(
+      NIL_ID, parent_task_id, 0, NIL_ACTOR_ID, 0, func_id, 2, 3, 11);
   task_args_add_ref(spec1, arg1);
   task_args_add_val(spec1, arg2, 11);
   finish_construct_task_spec(spec1);
 
   /* Construct a second identical task. */
-  task_spec *spec2 =
-      start_construct_task_spec(NIL_ID, parent_task_id, 0, func_id, 2, 3, 11);
+  task_spec *spec2 = start_construct_task_spec(
+      NIL_ID, parent_task_id, 0, NIL_ACTOR_ID, 0, func_id, 2, 3, 11);
   task_args_add_ref(spec2, arg1);
   task_args_add_val(spec2, arg2, 11);
   finish_construct_task_spec(spec2);
@@ -78,36 +78,37 @@ TEST deterministic_ids_test(void) {
   /* Create more tasks that are only mildly different. */
 
   /* Construct a task with a different parent task ID. */
-  task_spec *spec3 = start_construct_task_spec(NIL_ID, globally_unique_id(), 0,
-                                               func_id, 2, 3, 11);
+  task_spec *spec3 = start_construct_task_spec(
+      NIL_ID, globally_unique_id(), 0, NIL_ACTOR_ID, 0, func_id, 2, 3, 11);
   task_args_add_ref(spec3, arg1);
   task_args_add_val(spec3, arg2, 11);
   finish_construct_task_spec(spec3);
 
   /* Construct a task with a different parent counter. */
-  task_spec *spec4 =
-      start_construct_task_spec(NIL_ID, parent_task_id, 1, func_id, 2, 3, 11);
+  task_spec *spec4 = start_construct_task_spec(
+      NIL_ID, parent_task_id, 1, NIL_ACTOR_ID, 0, func_id, 2, 3, 11);
   task_args_add_ref(spec4, arg1);
   task_args_add_val(spec4, arg2, 11);
   finish_construct_task_spec(spec4);
 
   /* Construct a task with a different function ID. */
-  task_spec *spec5 = start_construct_task_spec(NIL_ID, parent_task_id, 0,
-                                               globally_unique_id(), 2, 3, 11);
+  task_spec *spec5 =
+      start_construct_task_spec(NIL_ID, parent_task_id, 0, NIL_ACTOR_ID, 0,
+                                globally_unique_id(), 2, 3, 11);
   task_args_add_ref(spec5, arg1);
   task_args_add_val(spec5, arg2, 11);
   finish_construct_task_spec(spec5);
 
   /* Construct a task with a different object ID argument. */
-  task_spec *spec6 =
-      start_construct_task_spec(NIL_ID, parent_task_id, 0, func_id, 2, 3, 11);
+  task_spec *spec6 = start_construct_task_spec(
+      NIL_ID, parent_task_id, 0, NIL_ACTOR_ID, 0, func_id, 2, 3, 11);
   task_args_add_ref(spec6, globally_unique_id());
   task_args_add_val(spec6, arg2, 11);
   finish_construct_task_spec(spec6);
 
   /* Construct a task with a different value argument. */
-  task_spec *spec7 =
-      start_construct_task_spec(NIL_ID, parent_task_id, 0, func_id, 2, 3, 11);
+  task_spec *spec7 = start_construct_task_spec(
+      NIL_ID, parent_task_id, 0, NIL_ACTOR_ID, 0, func_id, 2, 3, 11);
   task_args_add_ref(spec7, arg1);
   task_args_add_val(spec7, (uint8_t *) "hello_world", 11);
   finish_construct_task_spec(spec7);
@@ -148,8 +149,8 @@ TEST deterministic_ids_test(void) {
 TEST send_task(void) {
   task_id parent_task_id = globally_unique_id();
   function_id func_id = globally_unique_id();
-  task_spec *spec =
-      start_construct_task_spec(NIL_ID, parent_task_id, 0, func_id, 4, 2, 10);
+  task_spec *spec = start_construct_task_spec(
+      NIL_ID, parent_task_id, 0, NIL_ACTOR_ID, 0, func_id, 4, 2, 10);
   task_args_add_ref(spec, globally_unique_id());
   task_args_add_val(spec, (uint8_t *) "Hello", 5);
   task_args_add_val(spec, (uint8_t *) "World", 5);

src/common/test/test_common.h

@@ -22,8 +22,8 @@ static inline task_spec *example_task_spec_with_args(int64_t num_args,
   task_id parent_task_id = globally_unique_id();
   function_id func_id = globally_unique_id();
   task_spec *task =
-      start_construct_task_spec(NIL_ID, parent_task_id, 0, func_id, num_args,
-                                num_returns, arg_value_size);
+      start_construct_task_spec(NIL_ID, parent_task_id, 0, NIL_ACTOR_ID, 0,
+                                func_id, num_args, num_returns, arg_value_size);
   for (int64_t i = 0; i < num_args; ++i) {
     object_id arg_id;
     if (arg_ids == NULL) {