From d264713cebde1e37766ee5a406d7f56375dcc8d4 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Wed, 7 Sep 2016 18:46:04 -0700 Subject: [PATCH] Work around Arrow bug by increasing metadata size. (#415) --- lib/python/ray/worker.py | 4 ++-- thirdparty/arrow | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/python/ray/worker.py b/lib/python/ray/worker.py index 051d63c08..15a9dbc8d 100644 --- a/lib/python/ray/worker.py +++ b/lib/python/ray/worker.py @@ -348,7 +348,7 @@ class Worker(object): # in the following line, the "8" is for storing the metadata size, # the len(schema) is for storing the metadata and the 8192 is for storing # the metadata in the batch (see INITIAL_METADATA_SIZE in arrow) - size = size + 8 + len(schema) + 4096 + size = size + 8 + len(schema) + 4096 * 4 buff, segmentid = raylib.allocate_buffer(self.handle, objectid, size) # write the metadata length np.frombuffer(buff, dtype="int64", count=1)[0] = len(schema) @@ -371,7 +371,7 @@ class Worker(object): """ assert raylib.is_arrow(self.handle, objectid), "All objects should be serialized using Arrow." buff, segmentid, metadata_offset = raylib.get_buffer(self.handle, objectid) - metadata_size = np.frombuffer(buff, dtype="int64", count=1)[0] + metadata_size = int(np.frombuffer(buff, dtype="int64", count=1)[0]) metadata = np.frombuffer(buff, dtype="byte", offset=8, count=metadata_size) data = np.frombuffer(buff, dtype="byte")[8 + metadata_size:] serialized = libnumbuf.read_from_buffer(memoryview(data), bytearray(metadata), metadata_offset) diff --git a/thirdparty/arrow b/thirdparty/arrow index c781ef856..4622f4c5f 160000 --- a/thirdparty/arrow +++ b/thirdparty/arrow @@ -1 +1 @@ -Subproject commit c781ef8564714bb5149e8b5a311673f9162232ac +Subproject commit 4622f4c5fbec31f4d2ce49c90793b37c75a4a838