mirror of
https://github.com/wassname/ray.git
synced 2026-06-30 20:00:22 +08:00
Remove unused serialize/deserialize code in raylib. (#434)
This commit is contained in:
Robert Nishihara
committed by
Philipp Moritz
Philipp Moritz
parent
9ecf72484b
commit
a5c026c033
-264
@@ -222,270 +222,6 @@ void set_dict_item_and_transfer_ownership(PyObject* dict, PyObject* key, PyObjec
|
||||
Py_XDECREF(val);
|
||||
}
|
||||
|
||||
// Serialization
|
||||
|
||||
#define RAYLIB_SERIALIZE_NPY(TYPE, npy_type, proto_type) \
|
||||
case NPY_##TYPE: { \
|
||||
npy_type* buffer = (npy_type*) PyArray_DATA(array); \
|
||||
for (npy_intp i = 0; i < size; ++i) { \
|
||||
data->add_##proto_type##_data(buffer[i]); \
|
||||
} \
|
||||
} \
|
||||
break;
|
||||
|
||||
// serialize will serialize the python object val into the protocol buffer
|
||||
// object obj, returns 0 if successful and something else if not
|
||||
// NOTE: If some primitive types are added here, they may also need to be handled in serialization.py
|
||||
// FIXME(pcm): This currently only works for contiguous arrays
|
||||
// This method will push all of the object references contained in `obj` to the `objectids` vector.
|
||||
int serialize(PyObject* worker_capsule, PyObject* val, Obj* obj, std::vector<ObjectID> &objectids) {
|
||||
if (PyBool_Check(val)) {
|
||||
// The bool case must precede the int case because PyInt_Check passes for bools
|
||||
Bool* data = obj->mutable_bool_data();
|
||||
if (val == Py_False) {
|
||||
data->set_data(false);
|
||||
} else {
|
||||
data->set_data(true);
|
||||
}
|
||||
} else if (PyInt_Check(val)) {
|
||||
Int* data = obj->mutable_int_data();
|
||||
long d = PyInt_AsLong(val);
|
||||
data->set_data(d);
|
||||
} else if (PyLong_Check(val)) {
|
||||
// TODO(mehrdadn): We do not currently support arbitrary long values.
|
||||
int overflow = 0;
|
||||
Long* data = obj->mutable_long_data();
|
||||
data->set_data(PyLong_AsLongLongAndOverflow(val, &overflow));
|
||||
if (overflow) {
|
||||
PyErr_SetString(RayError, "serialization: long overflow");
|
||||
}
|
||||
} else if (PyFloat_Check(val)) {
|
||||
Double* data = obj->mutable_double_data();
|
||||
double d = PyFloat_AsDouble(val);
|
||||
data->set_data(d);
|
||||
} else if (PyTuple_Check(val)) {
|
||||
Tuple* data = obj->mutable_tuple_data();
|
||||
for (size_t i = 0, size = PyTuple_Size(val); i < size; ++i) {
|
||||
Obj* elem = data->add_elem();
|
||||
if (serialize(worker_capsule, PyTuple_GetItem(val, i), elem, objectids) != 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
} else if (PyList_Check(val)) {
|
||||
List* data = obj->mutable_list_data();
|
||||
for (size_t i = 0, size = PyList_Size(val); i < size; ++i) {
|
||||
Obj* elem = data->add_elem();
|
||||
if (serialize(worker_capsule, PyList_GetItem(val, i), elem, objectids) != 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
} else if (PyDict_Check(val)) {
|
||||
PyObject *pykey, *pyvalue;
|
||||
Py_ssize_t pos = 0;
|
||||
Dict* data = obj->mutable_dict_data();
|
||||
while (PyDict_Next(val, &pos, &pykey, &pyvalue)) {
|
||||
DictEntry* elem = data->add_elem();
|
||||
Obj* key = elem->mutable_key();
|
||||
if (serialize(worker_capsule, pykey, key, objectids) != 0) {
|
||||
return -1;
|
||||
}
|
||||
Obj* value = elem->mutable_value();
|
||||
if (serialize(worker_capsule, pyvalue, value, objectids) != 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
} else if (PyString_Check(val)) {
|
||||
char* buffer;
|
||||
Py_ssize_t length;
|
||||
PyString_AsStringAndSize(val, &buffer, &length); // creates pointer to internal buffer
|
||||
obj->mutable_string_data()->set_data(std::string(buffer, length));
|
||||
} else if (PyUnicode_Check(val)) {
|
||||
Py_ssize_t length;
|
||||
#if PY_MAJOR_VERSION >= 3
|
||||
char* data = PyUnicode_AsUTF8AndSize(val, &length); // TODO(pcm): Check if this is correct
|
||||
#else
|
||||
PyObject* str = PyUnicode_AsUTF8String(val);
|
||||
char* data = PyString_AS_STRING(str);
|
||||
length = PyString_GET_SIZE(str);
|
||||
#endif
|
||||
obj->mutable_unicode_data()->set_data(std::string(data, length));
|
||||
Py_XDECREF(str);
|
||||
} else if (val == Py_None) {
|
||||
obj->mutable_empty_data(); // allocate an Empty object, this is a None
|
||||
} else if (PyObject_IsInstance(val, (PyObject*) &PyObjectIDType)) {
|
||||
ObjectID objectid = ((PyObjectID*) val)->id;
|
||||
ObjID* data = obj->mutable_objectid_data();
|
||||
data->set_data(objectid);
|
||||
objectids.push_back(objectid);
|
||||
} else if (PyArray_Check(val) || PyArray_CheckScalar(val)) { // Python int and float already handled
|
||||
Array* data = obj->mutable_array_data();
|
||||
PyArrayObject* array; // will be deallocated at the end
|
||||
if (PyArray_IsScalar(val, Generic)) {
|
||||
data->set_is_scalar(true);
|
||||
PyArray_Descr* descr = PyArray_DescrFromScalar(val); // new reference
|
||||
array = (PyArrayObject*) PyArray_FromScalar(val, descr); // steals the new reference
|
||||
} else { // val is a numpy array
|
||||
array = PyArray_GETCONTIGUOUS((PyArrayObject*) val);
|
||||
}
|
||||
|
||||
npy_intp size = PyArray_SIZE(array);
|
||||
for (int i = 0; i < PyArray_NDIM(array); ++i) {
|
||||
data->add_shape(PyArray_DIM(array, i));
|
||||
}
|
||||
int typ = PyArray_TYPE(array);
|
||||
data->set_dtype(typ);
|
||||
switch (typ) {
|
||||
RAYLIB_SERIALIZE_NPY(FLOAT, npy_float, float)
|
||||
RAYLIB_SERIALIZE_NPY(DOUBLE, npy_double, double)
|
||||
RAYLIB_SERIALIZE_NPY(INT8, npy_int8, int)
|
||||
RAYLIB_SERIALIZE_NPY(INT16, npy_int16, int)
|
||||
RAYLIB_SERIALIZE_NPY(INT32, npy_int32, int)
|
||||
RAYLIB_SERIALIZE_NPY(INT64, npy_int64, int)
|
||||
RAYLIB_SERIALIZE_NPY(UINT8, npy_uint8, uint)
|
||||
RAYLIB_SERIALIZE_NPY(UINT16, npy_uint16, uint)
|
||||
RAYLIB_SERIALIZE_NPY(UINT32, npy_uint32, uint)
|
||||
RAYLIB_SERIALIZE_NPY(UINT64, npy_uint64, uint)
|
||||
case NPY_OBJECT: { // FIXME(pcm): Support arbitrary python objects, not only objectids
|
||||
PyArrayIterObject* iter = (PyArrayIterObject*) PyArray_IterNew((PyObject*)array);
|
||||
while (PyArray_ITER_NOTDONE(iter)) {
|
||||
PyObject** item = (PyObject**) PyArray_ITER_DATA(iter);
|
||||
ObjectID objectid;
|
||||
if (PyObject_IsInstance(*item, (PyObject*) &PyObjectIDType)) {
|
||||
objectid = ((PyObjectID*) (*item))->id;
|
||||
} else {
|
||||
std::stringstream ss;
|
||||
ss << "data type of " << PyString_AS_STRING(PyObject_Repr(*item))
|
||||
<< " not recognized";
|
||||
PyErr_SetString(PyExc_TypeError, ss.str().c_str());
|
||||
return -1;
|
||||
}
|
||||
data->add_objectid_data(objectid);
|
||||
objectids.push_back(objectid);
|
||||
PyArray_ITER_NEXT(iter);
|
||||
}
|
||||
Py_XDECREF(iter);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
PyErr_SetString(RayError, "serialization: numpy datatype not known");
|
||||
return -1;
|
||||
}
|
||||
Py_DECREF(array); // TODO(rkn): is this right?
|
||||
} else {
|
||||
std::stringstream ss;
|
||||
ss << "serialization: type of " << PyString_AS_STRING(PyObject_Repr(val))
|
||||
<< " not recognized";
|
||||
PyErr_SetString(RayError, ss.str().c_str());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define RAYLIB_DESERIALIZE_NPY(TYPE, npy_type, proto_type) \
|
||||
case NPY_##TYPE: { \
|
||||
npy_intp size = array.proto_type##_data_size(); \
|
||||
npy_type* buffer = (npy_type*) PyArray_DATA(pyarray); \
|
||||
for (npy_intp i = 0; i < size; ++i) { \
|
||||
buffer[i] = array.proto_type##_data(i); \
|
||||
} \
|
||||
} \
|
||||
break;
|
||||
|
||||
// This method will push all of the object references contained in `obj` to the `objectids` vector.
|
||||
static PyObject* deserialize(PyObject* worker_capsule, const Obj& obj, std::vector<ObjectID> &objectids) {
|
||||
if (obj.has_int_data()) {
|
||||
return PyInt_FromLong(obj.int_data().data());
|
||||
} else if (obj.has_long_data()) {
|
||||
return PyLong_FromLongLong(obj.long_data().data());
|
||||
} else if (obj.has_double_data()) {
|
||||
return PyFloat_FromDouble(obj.double_data().data());
|
||||
} else if (obj.has_bool_data()) {
|
||||
if (obj.bool_data().data()) {
|
||||
Py_RETURN_TRUE;
|
||||
} else {
|
||||
Py_RETURN_FALSE;
|
||||
}
|
||||
} else if (obj.has_tuple_data()) {
|
||||
const Tuple& data = obj.tuple_data();
|
||||
size_t size = data.elem_size();
|
||||
PyObject* tuple = PyTuple_New(size);
|
||||
for (size_t i = 0; i < size; ++i) {
|
||||
PyTuple_SetItem(tuple, i, deserialize(worker_capsule, data.elem(i), objectids));
|
||||
}
|
||||
return tuple;
|
||||
} else if (obj.has_list_data()) {
|
||||
const List& data = obj.list_data();
|
||||
size_t size = data.elem_size();
|
||||
PyObject* list = PyList_New(size);
|
||||
for (size_t i = 0; i < size; ++i) {
|
||||
PyList_SetItem(list, i, deserialize(worker_capsule, data.elem(i), objectids));
|
||||
}
|
||||
return list;
|
||||
} else if (obj.has_dict_data()) {
|
||||
const Dict& data = obj.dict_data();
|
||||
PyObject* dict = PyDict_New();
|
||||
size_t size = data.elem_size();
|
||||
for (size_t i = 0; i < size; ++i) {
|
||||
PyObject* pykey = deserialize(worker_capsule, data.elem(i).key(), objectids);
|
||||
PyObject* pyval = deserialize(worker_capsule, data.elem(i).value(), objectids);
|
||||
set_dict_item_and_transfer_ownership(dict, pykey, pyval);
|
||||
}
|
||||
return dict;
|
||||
} else if (obj.has_string_data()) {
|
||||
const char* buffer = obj.string_data().data().data();
|
||||
Py_ssize_t length = obj.string_data().data().size();
|
||||
return PyString_FromStringAndSize(buffer, length);
|
||||
} else if (obj.has_unicode_data()) {
|
||||
const char* buffer = obj.unicode_data().data().data();
|
||||
Py_ssize_t length = obj.unicode_data().data().size();
|
||||
return PyUnicode_FromStringAndSize(buffer, length);
|
||||
} else if (obj.has_empty_data()) {
|
||||
Py_RETURN_NONE;
|
||||
} else if (obj.has_objectid_data()) {
|
||||
objectids.push_back(obj.objectid_data().data());
|
||||
return make_pyobjectid(worker_capsule, obj.objectid_data().data());
|
||||
} else if (obj.has_array_data()) {
|
||||
const Array& array = obj.array_data();
|
||||
std::vector<npy_intp> dims;
|
||||
for (int i = 0; i < array.shape_size(); ++i) {
|
||||
dims.push_back(array.shape(i));
|
||||
}
|
||||
PyArrayObject* pyarray = (PyArrayObject*) PyArray_SimpleNew(array.shape_size(), dims.data(), array.dtype());
|
||||
switch (array.dtype()) {
|
||||
RAYLIB_DESERIALIZE_NPY(FLOAT, npy_float, float)
|
||||
RAYLIB_DESERIALIZE_NPY(DOUBLE, npy_double, double)
|
||||
RAYLIB_DESERIALIZE_NPY(INT8, npy_int8, int)
|
||||
RAYLIB_DESERIALIZE_NPY(INT16, npy_int16, int)
|
||||
RAYLIB_DESERIALIZE_NPY(INT32, npy_int32, int)
|
||||
RAYLIB_DESERIALIZE_NPY(INT64, npy_int64, int)
|
||||
RAYLIB_DESERIALIZE_NPY(UINT8, npy_uint8, uint)
|
||||
RAYLIB_DESERIALIZE_NPY(UINT16, npy_uint16, uint)
|
||||
RAYLIB_DESERIALIZE_NPY(UINT32, npy_uint32, uint)
|
||||
RAYLIB_DESERIALIZE_NPY(UINT64, npy_uint64, uint)
|
||||
case NPY_OBJECT: {
|
||||
npy_intp size = array.objectid_data_size();
|
||||
PyObject** buffer = (PyObject**) PyArray_DATA(pyarray);
|
||||
for (npy_intp i = 0; i < size; ++i) {
|
||||
buffer[i] = make_pyobjectid(worker_capsule, array.objectid_data(i));
|
||||
objectids.push_back(array.objectid_data(i));
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
PyErr_SetString(RayError, "deserialization: internal error (array type not implemented)");
|
||||
return NULL;
|
||||
}
|
||||
if (array.is_scalar()) {
|
||||
return PyArray_ScalarFromObject((PyObject*) pyarray);
|
||||
} else {
|
||||
return (PyObject*) pyarray;
|
||||
}
|
||||
} else {
|
||||
PyErr_SetString(RayError, "deserialization: internal error (type not implemented)");
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
// This converts an Python ObjectID to an Python integer.
|
||||
static PyObject* serialize_objectid(PyObject* self, PyObject* args) {
|
||||
Worker* worker;
|
||||
|
||||
Reference in New Issue
Block a user