Implement a first pass at actors in the API. (#242)

* Implement actor field for tasks

* Implement actor management in local scheduler.

* initial python frontend for actors

* import actors on worker

* IPython code completion and tests

* prepare creating actors through local schedulers

* add actor id to PyTask

* submit actor calls to local scheduler

* starting to integrate

* simple fix

* Fixes from rebasing.

* more work on python actors

* Improve local scheduler actor handlers.

* Pass actor ID to local scheduler when connecting a client.

* first working version of actors

* fixing actors

* fix creating two copies of the same actor

* fix actors

* remove sleep

* get rid of export synchronization

* update

* insert actor methods into the queue in the right order

* remove print statements

* make it compile again after rebase

* Minor updates.

* fix python actor ids

* Pass actor_id to start_worker.

* add test

* Minor changes.

* Update actor tests.

* Temporary plan for import counter.

* Temporarily fix import counters.

* Fix some tests.

* Fixes.

* Make actor creation non-blocking.

* Fix test?

* Fix actors on Python 2.

* fix rare case.

* Fix python 2 test.

* More tests.

* Small fixes.

* Linting.

* Revert tensorflow version to 0.12.0 temporarily.

* Small fix.

* Enhance inheritance test.
This commit is contained in:
Philipp Moritz
2017-02-15 00:10:05 -08:00
committed by Robert Nishihara
parent 072eadd57f
commit 12a68e84d2
32 changed files with 1812 additions and 117 deletions
+19 -5
View File
@@ -263,7 +263,13 @@ PyTypeObject PyObjectIDType = {
/* Define the PyTask class. */
static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
/* ID of the driver that this task originates from. */
unique_id driver_id;
/* ID of the actor this task should run on. */
unique_id actor_id = NIL_ACTOR_ID;
/* How many tasks have been launched on the actor so far? */
int actor_counter = 0;
/* ID of the function this task executes. */
function_id function_id;
/* Arguments of the task (can be PyObjectIDs or Python values). */
PyObject *arguments;
@@ -277,10 +283,11 @@ static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
int parent_counter;
/* Resource vector of the required resources to execute this task. */
PyObject *resource_vector = NULL;
if (!PyArg_ParseTuple(args, "O&O&OiO&i|O", &PyObjectToUniqueID, &driver_id,
if (!PyArg_ParseTuple(args, "O&O&OiO&i|O&iO", &PyObjectToUniqueID, &driver_id,
&PyObjectToUniqueID, &function_id, &arguments,
&num_returns, &PyObjectToUniqueID, &parent_task_id,
&parent_counter, &resource_vector)) {
&parent_counter, &PyObjectToUniqueID, &actor_id,
&actor_counter, &resource_vector)) {
return -1;
}
Py_ssize_t size = PyList_Size(arguments);
@@ -299,9 +306,9 @@ static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
}
/* Construct the task specification. */
int val_repr_index = 0;
self->spec = start_construct_task_spec(driver_id, parent_task_id,
parent_counter, function_id, size,
num_returns, value_data_bytes);
self->spec = start_construct_task_spec(
driver_id, parent_task_id, parent_counter, actor_id, actor_counter,
function_id, size, num_returns, value_data_bytes);
/* Add the task arguments. */
for (Py_ssize_t i = 0; i < size; ++i) {
PyObject *arg = PyList_GetItem(arguments, i);
@@ -350,6 +357,11 @@ static PyObject *PyTask_function_id(PyObject *self) {
return PyObjectID_make(function_id);
}
static PyObject *PyTask_actor_id(PyObject *self) {
actor_id actor_id = task_spec_actor_id(((PyTask *) self)->spec);
return PyObjectID_make(actor_id);
}
static PyObject *PyTask_driver_id(PyObject *self) {
unique_id driver_id = task_spec_driver_id(((PyTask *) self)->spec);
return PyObjectID_make(driver_id);
@@ -407,6 +419,8 @@ static PyObject *PyTask_returns(PyObject *self) {
static PyMethodDef PyTask_methods[] = {
{"function_id", (PyCFunction) PyTask_function_id, METH_NOARGS,
"Return the function ID for this task."},
{"actor_id", (PyCFunction) PyTask_actor_id, METH_NOARGS,
"Return the actor ID for this task."},
{"driver_id", (PyCFunction) PyTask_driver_id, METH_NOARGS,
"Return the driver ID for this task."},
{"task_id", (PyCFunction) PyTask_task_id, METH_NOARGS,