wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-16 11:27:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement Python global state API for xray. (#2125)

Browse Source 
* Implement global state API for xray.

* Fix object table.

* Fixes for log structure.

* Implement cluster_resources.

* Add driver task to task table.

* Remove python flatbuffers code

* Get some global state API tests running.

* Python linting.

* Fix linting.

* Fix mock modules for doc

* Copy over flatbuffer bindings.

* Fix for tests.

* Linting

* Fix monitor crash.
This commit is contained in:
Robert Nishihara
2018-05-29 16:25:54 -07:00
committed by Philipp Moritz
parent 166000b089
commit 6172f94c04
9 changed files with 474 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/lib/python/common_extension.cc
+33 -1
View File
@@ -10,7 +10,9 @@
#include "common.h"
#include "common_extension.h"
#include "common_protocol.h"
#include "ray/raylet/task.h"
#include "ray/raylet/task_spec.h"
#include "ray/raylet/task_execution_spec.h"
#include "task.h"
#include <string>
@@ -143,7 +145,15 @@ PyObject *PyTask_to_string(PyObject *self, PyObject *args) {
return NULL;
}
PyTask *task = (PyTask *) arg;
return PyBytes_FromStringAndSize((char *) task->spec, task->size);
if (!use_raylet(task)) {
return PyBytes_FromStringAndSize((char *) task->spec, task->size);
} else {
flatbuffers::FlatBufferBuilder fbb;
auto task_spec_string = task->task_spec->ToFlatbuffer(fbb);
fbb.Finish(task_spec_string);
return PyBytes_FromStringAndSize((char *) fbb.GetBufferPointer(),
fbb.GetSize());
}
}
static PyObject *PyObjectID_id(PyObject *self) {
@@ -693,6 +703,23 @@ static PyObject *PyTask_execution_dependencies_string(PyTask *self) {
fbb.GetSize());
}
static PyObject *PyTask_to_serialized_flatbuf(PyTask *self) {
RAY_CHECK(use_raylet(self));
const std::vector<ObjectID> execution_dependencies(
*self->execution_dependencies);
auto const execution_spec = ray::raylet::TaskExecutionSpecification(
std::move(execution_dependencies));
auto const task = ray::raylet::Task(execution_spec, *self->task_spec);
flatbuffers::FlatBufferBuilder fbb;
auto task_flatbuffer = task.ToFlatbuffer(fbb);
fbb.Finish(task_flatbuffer);
return PyBytes_FromStringAndSize(
reinterpret_cast<char *>(fbb.GetBufferPointer()), fbb.GetSize());
}
static PyMethodDef PyTask_methods[] = {
{"function_id", (PyCFunction) PyTask_function_id, METH_NOARGS,
"Return the function ID for this task."},
@@ -722,6 +749,11 @@ static PyMethodDef PyTask_methods[] = {
{"execution_dependencies_string",
(PyCFunction) PyTask_execution_dependencies_string, METH_NOARGS,
"Return the execution dependencies for the task as a string."},
{"_serialized_raylet_task", (PyCFunction) PyTask_to_serialized_flatbuf,
METH_NOARGS,
"This is a hack used to create a serialized flatbuffer object for the "
"driver task. We're doing this because creating the flatbuffer object in "
"Python didn't seem to work."},
{NULL} /* Sentinel */
};
src/ray/raylet/CMakeLists.txt
+11
View File
@@ -19,6 +19,17 @@ add_custom_command(
add_custom_target(gen_node_manager_fbs DEPENDS ${NODE_MANAGER_FBS_OUTPUT_FILES})
# Generate Python bindings for the flatbuffers objects.
set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
add_custom_command(
TARGET gen_node_manager_fbs
COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${NODE_MANAGER_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${NODE_MANAGER_FBS_SRC}"
VERBATIM)
add_dependencies(gen_node_manager_fbs flatbuffers_ep)
ADD_RAY_TEST(object_manager_integration_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main pthread ${Boost_SYSTEM_LIBRARY})
ADD_RAY_TEST(worker_pool_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY})