From 8bf877ac1ed0fddc10546dd7974598584956143f Mon Sep 17 00:00:00 2001
From: Philipp Moritz <pcmoritz@gmail.com>
Date: Thu, 4 Aug 2016 21:06:31 -0700
Subject: [PATCH] Serialize and Deserialize unicode (#349)

---
 lib/python/ray/serialization.py |  3 +++
 lib/python/ray/worker.py        |  2 ++
 protos/types.proto              |  5 +++++
 src/raylib.cc                   | 17 ++++++++++++++++-
 test/runtest.py                 |  2 +-
 thirdparty/arrow                |  2 +-
 thirdparty/numbuf               |  2 +-
 7 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/lib/python/ray/serialization.py b/lib/python/ray/serialization.py
index 4e10101f8..8cdbfa558 100644
--- a/lib/python/ray/serialization.py
+++ b/lib/python/ray/serialization.py
@@ -28,6 +28,9 @@ class Tuple(tuple):
 
 class Str(str):
   pass
+  
+class Unicode(unicode):
+  pass
 
 class NDArray(np.ndarray):
   pass
diff --git a/lib/python/ray/worker.py b/lib/python/ray/worker.py
index a089f4241..eb0e05672 100644
--- a/lib/python/ray/worker.py
+++ b/lib/python/ray/worker.py
@@ -489,6 +489,8 @@ class Worker(object):
       result = serialization.Tuple(result)
     elif isinstance(result, str):
       result = serialization.Str(result)
+    elif isinstance(result, unicode):
+      result = serialization.Unicode(result)
     elif isinstance(result, np.ndarray):
       result = result.view(serialization.NDArray)
     elif isinstance(result, np.generic):
diff --git a/protos/types.proto b/protos/types.proto
index 37bbe9da4..753902996 100644
--- a/protos/types.proto
+++ b/protos/types.proto
@@ -12,6 +12,10 @@ message String {
   string data = 1;
 }
 
+message Unicode {
+  string data = 1;
+}
+
 message Double {
   double data = 1;
 }
@@ -46,6 +50,7 @@ message ReusableVar {
 // Union of possible object types
 message Obj {
   String string_data = 1;
+  Unicode unicode_data = 13;
   Int int_data = 2;
   Long long_data = 12;
   Double double_data = 3;
diff --git a/src/raylib.cc b/src/raylib.cc
index 0b4849b6c..f30d0c600 100644
--- a/src/raylib.cc
+++ b/src/raylib.cc
@@ -289,6 +289,17 @@ int serialize(PyObject* worker_capsule, PyObject* val, Obj* obj, std::vector<Obj
     Py_ssize_t length;
     PyString_AsStringAndSize(val, &buffer, &length); // creates pointer to internal buffer
     obj->mutable_string_data()->set_data(buffer, length);
+  } else if (PyUnicode_Check(val)) {
+    Py_ssize_t length;
+    #if PY_MAJOR_VERSION >= 3
+      char* data = PyUnicode_AsUTF8AndSize(val, &length); // TODO(pcm): Check if this is correct
+    #else
+      PyObject* str = PyUnicode_AsUTF8String(val);
+      char* data = PyString_AS_STRING(str);
+      length = PyString_GET_SIZE(str);
+    #endif
+    obj->mutable_unicode_data()->set_data(data, length);
+    Py_XDECREF(str);
   } else if (val == Py_None) {
     obj->mutable_empty_data(); // allocate an Empty object, this is a None
   } else if (PyObject_IsInstance(val, (PyObject*) &PyObjectIDType)) {
@@ -343,7 +354,7 @@ int serialize(PyObject* worker_capsule, PyObject* val, Obj* obj, std::vector<Obj
         }
         break;
       default:
-        PyErr_SetString(RayError, "serialization: numpy datatype not know");
+        PyErr_SetString(RayError, "serialization: numpy datatype not known");
         return -1;
     }
     Py_DECREF(array); // TODO(rkn): is this right?
@@ -408,6 +419,10 @@ static PyObject* deserialize(PyObject* worker_capsule, const Obj& obj, std::vect
     const char* buffer = obj.string_data().data().data();
     Py_ssize_t length = obj.string_data().data().size();
     return PyString_FromStringAndSize(buffer, length);
+  } else if (obj.has_unicode_data()) {
+    const char* buffer = obj.unicode_data().data().data();
+    Py_ssize_t length = obj.unicode_data().data().size();
+    return PyUnicode_FromStringAndSize(buffer, length);
   } else if (obj.has_empty_data()) {
     Py_RETURN_NONE;
   } else if (obj.has_objectid_data()) {
diff --git a/test/runtest.py b/test/runtest.py
index d29127b4a..0eacc5014 100644
--- a/test/runtest.py
+++ b/test/runtest.py
@@ -13,7 +13,7 @@ import ray.array.distributed as da
 
 RAY_TEST_OBJECTS = [[1, "hello", 3.0], 42, 43L, "hello world", 42.0, 1L << 62,
                     (1.0, "hi"), None, (None, None), ("hello", None),
-                    True, False, (True, False),
+                    True, False, (True, False), u"\u262F",
                     {True: "hello", False: "world"},
                     {"hello" : "world", 1: 42, 1.0: 45}, {},
                     np.int8(3), np.int32(4), np.int64(5),
diff --git a/thirdparty/arrow b/thirdparty/arrow
index ab67f8481..c781ef856 160000
--- a/thirdparty/arrow
+++ b/thirdparty/arrow
@@ -1 +1 @@
-Subproject commit ab67f84810d4343e7d75142fb80754a4d33cc0a0
+Subproject commit c781ef8564714bb5149e8b5a311673f9162232ac
diff --git a/thirdparty/numbuf b/thirdparty/numbuf
index a81dc0c54..8e165d43d 160000
--- a/thirdparty/numbuf
+++ b/thirdparty/numbuf
@@ -1 +1 @@
-Subproject commit a81dc0c54197116dd236f4476ed13bdc6ee5e55b
+Subproject commit 8e165d43d455925732d5361b79f818b6cead59ca