Serialize and Deserialize unicode (#349)

lib/python/ray/serialization.py

@@ -28,6 +28,9 @@ class Tuple(tuple):
 
 class Str(str):
   pass
+  
+class Unicode(unicode):
+  pass
 
 class NDArray(np.ndarray):
   pass

lib/python/ray/worker.py

@@ -489,6 +489,8 @@ class Worker(object):
       result = serialization.Tuple(result)
     elif isinstance(result, str):
       result = serialization.Str(result)
+    elif isinstance(result, unicode):
+      result = serialization.Unicode(result)
     elif isinstance(result, np.ndarray):
       result = result.view(serialization.NDArray)
     elif isinstance(result, np.generic):

protos/types.proto

@@ -12,6 +12,10 @@ message String {
   string data = 1;
 }
 
+message Unicode {
+  string data = 1;
+}
+
 message Double {
   double data = 1;
 }
@@ -46,6 +50,7 @@ message ReusableVar {
 // Union of possible object types
 message Obj {
   String string_data = 1;
+  Unicode unicode_data = 13;
   Int int_data = 2;
   Long long_data = 12;
   Double double_data = 3;

src/raylib.cc

@@ -289,6 +289,17 @@ int serialize(PyObject* worker_capsule, PyObject* val, Obj* obj, std::vector<Obj
     Py_ssize_t length;
     PyString_AsStringAndSize(val, &buffer, &length); // creates pointer to internal buffer
     obj->mutable_string_data()->set_data(buffer, length);
+  } else if (PyUnicode_Check(val)) {
+    Py_ssize_t length;
+    #if PY_MAJOR_VERSION >= 3
+      char* data = PyUnicode_AsUTF8AndSize(val, &length); // TODO(pcm): Check if this is correct
+    #else
+      PyObject* str = PyUnicode_AsUTF8String(val);
+      char* data = PyString_AS_STRING(str);
+      length = PyString_GET_SIZE(str);
+    #endif
+    obj->mutable_unicode_data()->set_data(data, length);
+    Py_XDECREF(str);
   } else if (val == Py_None) {
     obj->mutable_empty_data(); // allocate an Empty object, this is a None
   } else if (PyObject_IsInstance(val, (PyObject*) &PyObjectIDType)) {
@@ -343,7 +354,7 @@ int serialize(PyObject* worker_capsule, PyObject* val, Obj* obj, std::vector<Obj
         }
         break;
       default:
-        PyErr_SetString(RayError, "serialization: numpy datatype not know");
+        PyErr_SetString(RayError, "serialization: numpy datatype not known");
         return -1;
     }
     Py_DECREF(array); // TODO(rkn): is this right?
@@ -408,6 +419,10 @@ static PyObject* deserialize(PyObject* worker_capsule, const Obj& obj, std::vect
     const char* buffer = obj.string_data().data().data();
     Py_ssize_t length = obj.string_data().data().size();
     return PyString_FromStringAndSize(buffer, length);
+  } else if (obj.has_unicode_data()) {
+    const char* buffer = obj.unicode_data().data().data();
+    Py_ssize_t length = obj.unicode_data().data().size();
+    return PyUnicode_FromStringAndSize(buffer, length);
   } else if (obj.has_empty_data()) {
     Py_RETURN_NONE;
   } else if (obj.has_objectid_data()) {

test/runtest.py

@@ -13,7 +13,7 @@ import ray.array.distributed as da
 
 RAY_TEST_OBJECTS = [[1, "hello", 3.0], 42, 43L, "hello world", 42.0, 1L << 62,
                     (1.0, "hi"), None, (None, None), ("hello", None),
-                    True, False, (True, False),
+                    True, False, (True, False), u"\u262F",
                     {True: "hello", False: "world"},
                     {"hello" : "world", 1: 42, 1.0: 45}, {},
                     np.int8(3), np.int32(4), np.int64(5),