From 674ec3a3cb88300be7969c853820628f3194dbd9 Mon Sep 17 00:00:00 2001 From: Alexey Tumanov Date: Sun, 8 Jan 2017 02:16:40 -0800 Subject: [PATCH] generate pytask from string and string from pytask (#188) * pytask creation from bytestring: saving work * pytask now works * documentation and tests * linting * Lint and fix test case --- src/common/lib/python/common_extension.c | 49 ++++++++++++++++++++++++ src/common/lib/python/common_extension.h | 3 ++ src/common/lib/python/common_module.c | 5 +++ src/common/test/test.py | 26 ++++++++----- src/photon/photon_extension.c | 5 +++ 5 files changed, 78 insertions(+), 10 deletions(-) diff --git a/src/common/lib/python/common_extension.c b/src/common/lib/python/common_extension.c index 7ea1941c8..19f33afd9 100644 --- a/src/common/lib/python/common_extension.c +++ b/src/common/lib/python/common_extension.c @@ -68,6 +68,55 @@ PyObject *PyObjectID_make(object_id object_id) { return (PyObject *) result; } +/** + * Convert a string to a Ray task specification Python object. + * + * This is called from Python like + * + * task = photon.task_from_string("...") + * + * @param task_string String representation of the task specification. + * @return Python task specification object. + */ +PyObject *PyTask_from_string(PyObject *self, PyObject *args) { + const char *data; + int size; + if (!PyArg_ParseTuple(args, "s#", &data, &size)) { + return NULL; + } + PyTask *result = PyObject_New(PyTask, &PyTaskType); + result = (PyTask *) PyObject_Init((PyObject *) result, &PyTaskType); + result->spec = malloc(size); + memcpy(result->spec, data, size); + /* TODO(pcm): Better error checking once we use flatbuffers. */ + if (size != task_spec_size(result->spec)) { + PyErr_SetString(CommonError, + "task_from_string: task specification string malformed"); + return NULL; + } + return (PyObject *) result; +} + +/** + * Convert a Ray task specification Python object to a string. + * + * This is called from Python like + * + * s = photon.task_to_string(task) + * + * @param task Ray task specification Python object. + * @return String representing the task specification. + */ +PyObject *PyTask_to_string(PyObject *self, PyObject *args) { + PyObject *arg; + if (!PyArg_ParseTuple(args, "O", &arg)) { + return NULL; + } + PyTask *task = (PyTask *) arg; + return PyBytes_FromStringAndSize((char *) task->spec, + task_spec_size(task->spec)); +} + static PyObject *PyObjectID_id(PyObject *self) { PyObjectID *s = (PyObjectID *) self; return PyBytes_FromStringAndSize((char *) &s->object_id.id[0], diff --git a/src/common/lib/python/common_extension.h b/src/common/lib/python/common_extension.h index 3fc03f78c..9a5cfe851 100644 --- a/src/common/lib/python/common_extension.h +++ b/src/common/lib/python/common_extension.h @@ -39,6 +39,9 @@ PyObject *PyObjectID_make(object_id object_id); PyObject *check_simple_value(PyObject *self, PyObject *args); +PyObject *PyTask_to_string(PyObject *, PyObject *args); +PyObject *PyTask_from_string(PyObject *, PyObject *args); + PyObject *compute_put_id(PyObject *self, PyObject *args); PyObject *PyTask_make(task_spec *task_spec); diff --git a/src/common/lib/python/common_module.c b/src/common/lib/python/common_module.c index 1ee0e3f50..3b204f4a2 100644 --- a/src/common/lib/python/common_module.c +++ b/src/common/lib/python/common_module.c @@ -7,6 +7,11 @@ static PyMethodDef common_methods[] = { "Should the object be passed by value?"}, {"compute_put_id", compute_put_id, METH_VARARGS, "Return the object ID for a put call within a task."}, + {"task_from_string", PyTask_from_string, METH_VARARGS, + "Creates a Python PyTask object from a string representation of " + "task_spec."}, + {"task_to_string", PyTask_to_string, METH_VARARGS, + "Translates a PyTask python object to a byte string."}, {NULL} /* Sentinel */ }; diff --git a/src/common/test/test.py b/src/common/test/test.py index d4683ffec..d38d1dba5 100644 --- a/src/common/test/test.py +++ b/src/common/test/test.py @@ -112,7 +112,18 @@ class TestObjectID(unittest.TestCase): class TestTask(unittest.TestCase): - def test_create_task(self): + def check_task(self, task, function_id, num_return_vals, args): + self.assertEqual(function_id.id(), task.function_id().id()) + retrieved_args = task.arguments() + self.assertEqual(num_return_vals, len(task.returns())) + self.assertEqual(len(args), len(retrieved_args)) + for i in range(len(retrieved_args)): + if isinstance(retrieved_args[i], common.ObjectID): + self.assertEqual(retrieved_args[i].id(), args[i].id()) + else: + self.assertEqual(retrieved_args[i], args[i]) + + def test_create_and_serialize_task(self): # TODO(rkn): The function ID should be a FunctionID object, not an ObjectID. parent_id = random_task_id() function_id = random_function_id() @@ -146,15 +157,10 @@ class TestTask(unittest.TestCase): for args in args_list: for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: task = common.Task(function_id, args, num_return_vals, parent_id, 0) - self.assertEqual(function_id.id(), task.function_id().id()) - retrieved_args = task.arguments() - self.assertEqual(num_return_vals, len(task.returns())) - self.assertEqual(len(args), len(retrieved_args)) - for i in range(len(retrieved_args)): - if isinstance(retrieved_args[i], common.ObjectID): - self.assertEqual(retrieved_args[i].id(), args[i].id()) - else: - self.assertEqual(retrieved_args[i], args[i]) + self.check_task(task, function_id, num_return_vals, args) + data = common.task_to_string(task) + task2 = common.task_from_string(data) + self.check_task(task2, function_id, num_return_vals, args) if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/src/photon/photon_extension.c b/src/photon/photon_extension.c index 8ffe3ac6b..7b9f752d1 100644 --- a/src/photon/photon_extension.c +++ b/src/photon/photon_extension.c @@ -135,6 +135,11 @@ static PyMethodDef photon_methods[] = { "Should the object be passed by value?"}, {"compute_put_id", compute_put_id, METH_VARARGS, "Return the object ID for a put call within a task."}, + {"task_from_string", PyTask_from_string, METH_VARARGS, + "Creates a Python PyTask object from a string representation of " + "task_spec."}, + {"task_to_string", PyTask_to_string, METH_VARARGS, + "Translates a PyTask python object to a byte string."}, {NULL} /* Sentinel */ };