wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-16 11:27:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Reintroduce passing arguments by value to remote functions. (#425)

Browse Source 
* Reintroduce passing arguments by value to remote functions.

* Check size of arguments passed by value.

* Fix computation graph visualization.
This commit is contained in:
Robert Nishihara
2016-09-10 21:11:18 -07:00
committed by Philipp Moritz
parent 0191d42751
commit ba56b08474
7 changed files with 203 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/raylib.cc
+22 -16
View File
@@ -578,9 +578,21 @@ static PyObject* serialize_task(PyObject* self, PyObject* args) {
if (PyList_Check(arguments)) {
for (size_t i = 0, size = PyList_Size(arguments); i < size; ++i) {
PyObject* element = PyList_GetItem(arguments, i);
ObjectID objectid = ((PyObjectID*) element)->id;
task->add_arg(objectid);
objectids.push_back(objectid);
if (PyObject_IsInstance(element, (PyObject*)&PyObjectIDType)) {
// Handle the case where the argument to the task is an ObjectID.
ObjectID objectid = ((PyObjectID*) element)->id;
task->add_arg()->set_objectid(objectid);
objectids.push_back(objectid);
} else if (PyString_CheckExact(element)) {
// Handle the case where the argument to the task is being passed by
// value and we receive an argument serialized as a string here.
char* buffer;
Py_ssize_t length;
PyString_AsStringAndSize(element, &buffer, &length);
task->add_arg()->set_serialized_arg(std::string(buffer, length));
} else {
RAY_CHECK(false, "This code should be unreachable.");
}
}
} else {
PyErr_SetString(RayError, "serialize_task: second argument needs to be a list");
@@ -595,17 +607,6 @@ static PyObject* serialize_task(PyObject* self, PyObject* args) {
std::string output;
task->SerializeToString(&output);
int task_size = output.length();
if (task_size > 1024) {
// Large objects should not be passed to tasks by value. Instead, they
// should be placed in the object store and passed by object
// reference.
RAY_LOG(RAY_INFO, "Warning: attempting to serialize a task with size " << task_size << ".");
PyErr_SetString(RaySizeError, "serialize_task: This task is too large (greater than 1024 bytes). "
"Please do not pass large objects by value to remote functions. "
"Instead, put large objects in the object store and pass them by "
"object reference to the remote function.");
return NULL;
}
return PyCapsule_New(static_cast<void*>(task), "task", &TaskCapsule_Destructor);
}
@@ -615,8 +616,13 @@ static PyObject* deserialize_task(PyObject* worker_capsule, const Task& task) {
int argsize = task.arg_size();
PyObject* arglist = PyList_New(argsize);
for (int i = 0; i < argsize; ++i) {
PyList_SetItem(arglist, i, make_pyobjectid(worker_capsule, task.arg(i)));
objectids.push_back(task.arg(i));
if (task.arg(i).serialized_arg().empty()) {
PyList_SetItem(arglist, i, make_pyobjectid(worker_capsule, task.arg(i).objectid()));
objectids.push_back(task.arg(i).objectid());
} else {
PyObject* serialized_arg = PyString_FromStringAndSize(task.arg(i).serialized_arg().data(), task.arg(i).serialized_arg().size());
PyList_SetItem(arglist, i, serialized_arg);
}
}
Worker* worker;
PyObjectToWorker(worker_capsule, &worker);
src/scheduler.cc
+28 -22
View File
@@ -690,13 +690,15 @@ void SchedulerService::assign_task(OperationId operationid, WorkerId workerid, c
AckReply reply;
RAY_LOG(RAY_INFO, "starting to send arguments");
for (size_t i = 0; i < task.arg_size(); ++i) {
ObjectID objectid = task.arg(i);
ObjectID canonical_objectid = get_canonical_objectid(objectid);
// Notify the relevant objstore about potential aliasing when it's ready
GET(alias_notification_queue_)->push_back(std::make_pair(objstoreid, std::make_pair(objectid, canonical_objectid)));
attempt_notify_alias(objstoreid, objectid, canonical_objectid);
RAY_LOG(RAY_DEBUG, "task contains object ref " << canonical_objectid);
deliver_object_async_if_necessary(canonical_objectid, pick_objstore(canonical_objectid), objstoreid);
if (task.arg(i).serialized_arg().empty()) {
ObjectID objectid = task.arg(i).objectid();
ObjectID canonical_objectid = get_canonical_objectid(objectid);
// Notify the relevant objstore about potential aliasing when it's ready
GET(alias_notification_queue_)->push_back(std::make_pair(objstoreid, std::make_pair(objectid, canonical_objectid)));
attempt_notify_alias(objstoreid, objectid, canonical_objectid);
RAY_LOG(RAY_DEBUG, "task contains object ref " << canonical_objectid);
deliver_object_async_if_necessary(canonical_objectid, pick_objstore(canonical_objectid), objstoreid);
}
}
{
auto workers = GET(workers_);
@@ -709,13 +711,15 @@ void SchedulerService::assign_task(OperationId operationid, WorkerId workerid, c
bool SchedulerService::can_run(const Task& task) {
auto objtable = GET(objtable_);
for (int i = 0; i < task.arg_size(); ++i) {
ObjectID objectid = task.arg(i);
if (!has_canonical_objectid(objectid)) {
return false;
}
ObjectID canonical_objectid = get_canonical_objectid(objectid);
if (canonical_objectid >= objtable->size() || (*objtable)[canonical_objectid].size() == 0) {
return false;
if (task.arg(i).serialized_arg().empty()) {
ObjectID objectid = task.arg(i).objectid();
if (!has_canonical_objectid(objectid)) {
return false;
}
ObjectID canonical_objectid = get_canonical_objectid(objectid);
if (canonical_objectid >= objtable->size() || (*objtable)[canonical_objectid].size() == 0) {
return false;
}
}
}
return true;
@@ -952,14 +956,16 @@ void SchedulerService::schedule_tasks_location_aware() {
// determine how many objects would need to be shipped
size_t num_shipped_objects = 0;
for (int j = 0; j < task.arg_size(); ++j) {
ObjectID objectid = task.arg(j);
RAY_CHECK(has_canonical_objectid(objectid), "no canonical object ref found even though task is ready; that should not be possible!");
ObjectID canonical_objectid = get_canonical_objectid(objectid);
{
// check if the object is already in the local object store
auto objtable = GET(objtable_);
if (!std::binary_search((*objtable)[canonical_objectid].begin(), (*objtable)[canonical_objectid].end(), objstoreid)) {
num_shipped_objects += 1;
if (task.arg(j).serialized_arg().empty()) {
ObjectID objectid = task.arg(j).objectid();
RAY_CHECK(has_canonical_objectid(objectid), "no canonical object ref found even though task is ready; that should not be possible!");
ObjectID canonical_objectid = get_canonical_objectid(objectid);
{
// check if the object is already in the local object store
auto objtable = GET(objtable_);
if (!std::binary_search((*objtable)[canonical_objectid].begin(), (*objtable)[canonical_objectid].end(), objstoreid)) {
num_shipped_objects += 1;
}
}
}
}