wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-15 11:25:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python 3 compatibility. (#121)

Browse Source 
* Make common module Python 3 compatible.

* Make plasma module Python 3 compatible.

* Make photon module Python 3 compatible.

* Make numbuf module Python 3 compatible.

* Remaining changes for Python 3 compatibility.

* Test Python 3 in Travis.

* Fixes.
This commit is contained in:
Robert Nishihara
2016-12-16 14:40:37 -08:00
parent 946242929f
commit 79dd1815a2
42 changed files with 652 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files
numbuf/python/src/pynumbuf/adapters/python.cc
+15 -10
View File
@@ -16,6 +16,10 @@ extern PyObject* numbuf_deserialize_callback;
namespace numbuf {
#if PY_MAJOR_VERSION >= 3
#define PyInt_FromLong PyLong_FromLong
#endif
Status get_value(
ArrayPtr arr, int32_t index, int32_t type, PyObject* base, PyObject** result) {
switch (arr->type()->type) {
@@ -30,7 +34,7 @@ Status get_value(
int32_t nchars;
const uint8_t* str =
std::static_pointer_cast<BinaryArray>(arr)->GetValue(index, &nchars);
*result = PyString_FromStringAndSize(reinterpret_cast<const char*>(str), nchars);
*result = PyBytes_FromStringAndSize(reinterpret_cast<const char*>(str), nchars);
return Status::OK();
}
case Type::STRING: {
@@ -82,24 +86,25 @@ Status append(PyObject* elem, SequenceBuilder& builder, std::vector<PyObject*>&
int64_t data = PyLong_AsLongLongAndOverflow(elem, &overflow);
RETURN_NOT_OK(builder.AppendInt64(data));
if (overflow) { return Status::NotImplemented("long overflow"); }
#if PY_MAJOR_VERSION < 3
} else if (PyInt_Check(elem)) {
RETURN_NOT_OK(builder.AppendInt64(static_cast<int64_t>(PyInt_AS_LONG(elem))));
} else if (PyString_Check(elem)) {
auto data = reinterpret_cast<uint8_t*>(PyString_AS_STRING(elem));
auto size = PyString_GET_SIZE(elem);
#endif
} else if (PyBytes_Check(elem)) {
auto data = reinterpret_cast<uint8_t*>(PyBytes_AS_STRING(elem));
auto size = PyBytes_GET_SIZE(elem);
RETURN_NOT_OK(builder.AppendBytes(data, size));
} else if (PyUnicode_Check(elem)) {
Py_ssize_t size;
#if PY_MAJOR_VERSION >= 3
char* data =
PyUnicode_AsUTF8AndSize(elem, &size); // TODO(pcm): Check if this is correct
char* data = PyUnicode_AsUTF8AndSize(elem, &size);
#else
PyObject* str = PyUnicode_AsUTF8String(elem);
char* data = PyString_AS_STRING(str);
size = PyString_GET_SIZE(str);
Py_XDECREF(str);
#endif
Status s = builder.AppendString(data, size);
Py_XDECREF(str);
RETURN_NOT_OK(s);
} else if (PyList_Check(elem)) {
builder.AppendList(PyList_Size(elem));
@@ -119,7 +124,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, std::vector<PyObject*>&
} else {
if (!numbuf_serialize_callback) {
std::stringstream ss;
ss << "data type of " << PyString_AS_STRING(PyObject_Repr(elem))
ss << "data type of " << PyBytes_AS_STRING(PyObject_Repr(elem))
<< " not recognized and custom serialization handler not registered";
return Status::NotImplemented(ss.str());
} else {
@@ -246,7 +251,7 @@ Status SerializeDict(
// This block is used to decrement the reference counts of the results
// returned by the serialization callback, which is called in SerializeArray
// in numpy.cc as well as in DeserializeDict and in append in this file.
static PyObject* py_type = PyString_FromString("_pytype_");
static PyObject* py_type = PyUnicode_FromString("_pytype_");
for (const auto& dict : dicts) {
if (PyDict_Contains(dict, py_type)) {
// If the dictionary contains the key "_pytype_", then the user has to
@@ -273,7 +278,7 @@ Status DeserializeDict(std::shared_ptr<Array> array, int32_t start_idx, int32_t
}
Py_XDECREF(keys); // PyList_GetItem(keys, ...) incremented the reference count
Py_XDECREF(vals); // PyList_GetItem(vals, ...) incremented the reference count
static PyObject* py_type = PyString_FromString("_pytype_");
static PyObject* py_type = PyUnicode_FromString("_pytype_");
if (PyDict_Contains(result, py_type) && numbuf_deserialize_callback) {
PyObject* arglist = Py_BuildValue("(O)", result);
// The result of the call to PyObject_CallObject will be passed to Python
numbuf/python/src/pynumbuf/numbuf.cc
+56 -11
View File
@@ -5,6 +5,8 @@
#define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API
#include <numpy/arrayobject.h>
#include "bytesobject.h"
#include <iostream>
#include <arrow/ipc/metadata.h>
@@ -72,7 +74,7 @@ static PyObject* serialize_list(PyObject* self, PyObject* args) {
PyObject* r = PyTuple_New(3);
PyTuple_SetItem(r, 0, PyByteArray_FromStringAndSize(ptr, buffer->size()));
PyTuple_SetItem(r, 1, PyInt_FromLong(size));
PyTuple_SetItem(r, 1, PyLong_FromLong(size));
PyTuple_SetItem(r, 2,
PyCapsule_New(reinterpret_cast<void*>(batch), "arrow", &ArrowCapsule_Destructor));
return r;
@@ -95,20 +97,22 @@ static PyObject* write_to_buffer(PyObject* self, PyObject* args) {
int64_t header_end_offset;
ARROW_CHECK_OK(ipc::WriteRecordBatch((*batch)->columns(), (*batch)->num_rows(),
target.get(), &body_end_offset, &header_end_offset));
return PyInt_FromLong(header_end_offset);
return PyLong_FromLong(header_end_offset);
}
/* Documented in doc/numbuf.rst in ray-core */
static PyObject* read_from_buffer(PyObject* self, PyObject* args) {
PyObject* memoryview;
PyObject* metadata;
PyObject* data_memoryview;
PyObject* metadata_memoryview;
int64_t metadata_offset;
if (!PyArg_ParseTuple(args, "OOL", &memoryview, &metadata, &metadata_offset)) {
if (!PyArg_ParseTuple(
args, "OOL", &data_memoryview, &metadata_memoryview, &metadata_offset)) {
return NULL;
}
auto ptr = reinterpret_cast<uint8_t*>(PyByteArray_AsString(metadata));
auto schema_buffer = std::make_shared<Buffer>(ptr, PyByteArray_Size(metadata));
Py_buffer* metadata_buffer = PyMemoryView_GET_BUFFER(metadata_memoryview);
auto ptr = reinterpret_cast<uint8_t*>(metadata_buffer->buf);
auto schema_buffer = std::make_shared<Buffer>(ptr, metadata_buffer->len);
std::shared_ptr<ipc::Message> message;
ARROW_CHECK_OK(ipc::Message::Open(schema_buffer, &message));
DCHECK_EQ(ipc::Message::SCHEMA, message->type());
@@ -116,7 +120,7 @@ static PyObject* read_from_buffer(PyObject* self, PyObject* args) {
std::shared_ptr<Schema> schema;
ARROW_CHECK_OK(schema_msg->GetSchema(&schema));
Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview);
Py_buffer* buffer = PyMemoryView_GET_BUFFER(data_memoryview);
auto source = std::make_shared<FixedBufferStream>(
reinterpret_cast<uint8_t*>(buffer->buf), buffer->len);
std::shared_ptr<arrow::ipc::RecordBatchReader> reader;
@@ -180,13 +184,54 @@ static PyMethodDef NumbufMethods[] = {
"set serialization and deserialization callbacks"},
{NULL, NULL, 0, NULL}};
PyMODINIT_FUNC initlibnumbuf(void) {
PyObject* m;
m = Py_InitModule3("libnumbuf", NumbufMethods, "Python C Extension for Numbuf");
// clang-format off
#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
"libnumbuf", /* m_name */
"Python C Extension for Numbuf", /* m_doc */
0, /* m_size */
NumbufMethods, /* m_methods */
NULL, /* m_reload */
NULL, /* m_traverse */
NULL, /* m_clear */
NULL, /* m_free */
};
#endif
// clang-format on
#if PY_MAJOR_VERSION >= 3
#define INITERROR return NULL
#else
#define INITERROR return
#endif
#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
#define PyMODINIT_FUNC void
#endif
#if PY_MAJOR_VERSION >= 3
#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
#else
#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
#endif
MOD_INIT(libnumbuf) {
#if PY_MAJOR_VERSION >= 3
PyObject* m = PyModule_Create(&moduledef);
#else
PyObject* m =
Py_InitModule3("libnumbuf", NumbufMethods, "Python C Extension for Numbuf");
#endif
char numbuf_error[] = "numbuf.error";
NumbufError = PyErr_NewException(numbuf_error, NULL, NULL);
Py_INCREF(NumbufError);
PyModule_AddObject(m, "numbuf_error", NumbufError);
import_array();
#if PY_MAJOR_VERSION >= 3
return m;
#endif
}
}
numbuf/python/test/runtest.py
+7 -3
View File
@@ -6,10 +6,11 @@ import unittest
import numbuf
import numpy as np
from numpy.testing import assert_equal
import sys
TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world",
TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43, "hello world",
u"x", u"\u262F", 42.0,
1L << 62, (1.0, "hi"),
1 << 62, (1.0, "hi"),
None, (None, None), ("hello", None),
True, False, (True, False), "hello",
{True: "hello", False: "world"},
@@ -18,6 +19,9 @@ TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world"
np.uint8(3), np.uint32(4), np.uint64(5),
np.float32(1.0), np.float64(1.0)]
if sys.version_info < (3, 0):
TEST_OBJECTS += [long(42), long(1 << 62)]
class SerializationTests(unittest.TestCase):
def roundTripTest(self, data):
@@ -110,7 +114,7 @@ class SerializationTests(unittest.TestCase):
size = size + 4096 # INITIAL_METADATA_SIZE in arrow
buff = np.zeros(size, dtype="uint8")
metadata_offset = numbuf.write_to_buffer(batch, memoryview(buff))
array = numbuf.read_from_buffer(memoryview(buff), schema, metadata_offset)
array = numbuf.read_from_buffer(memoryview(buff), memoryview(schema), metadata_offset)
result = numbuf.deserialize_list(array)
assert_equal(result[0], obj)