From 09a3ff717377e3c11bf2fe8a0fdfd10116490682 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Fri, 28 Oct 2016 14:30:20 -0700 Subject: [PATCH] Pip install numbuf. (#8) --- install-dependencies.sh | 2 ++ lib/python/ray/serialization.py | 4 ++-- lib/python/ray/worker.py | 16 ++++++++-------- test/runtest.py | 1 - 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/install-dependencies.sh b/install-dependencies.sh index 1092b4bb6..d944caaf8 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -41,3 +41,5 @@ elif [[ $platform == "macosx" ]]; then sudo pip install numpy funcsigs subprocess32 protobuf colorama graphviz --ignore-installed six sudo pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4 # We use the latest version of cloudpickle because it can serialize named tuples. fi + +sudo pip install --upgrade git+git://github.com/ray-project/numbuf.git@d1974afbab9f0f1bcf8af15a8c476d868ad31aff diff --git a/lib/python/ray/serialization.py b/lib/python/ray/serialization.py index bff61c3f0..1f531e674 100644 --- a/lib/python/ray/serialization.py +++ b/lib/python/ray/serialization.py @@ -1,7 +1,7 @@ import numpy as np import pickling import libraylib as raylib -import libnumbuf +import numbuf def is_argument_serializable(value): """Checks if value is a composition of primitive types. @@ -216,4 +216,4 @@ def deserialize(serialized_obj): return obj # Register the callbacks with numbuf. -libnumbuf.register_callbacks(serialize, deserialize) +numbuf.register_callbacks(serialize, deserialize) diff --git a/lib/python/ray/worker.py b/lib/python/ray/worker.py index 4a6e4189e..4f9abadd0 100644 --- a/lib/python/ray/worker.py +++ b/lib/python/ray/worker.py @@ -19,7 +19,7 @@ import serialization import internal.graph_pb2 import graph import services -import libnumbuf +import numbuf import libraylib as raylib contained_objectids = [] @@ -38,7 +38,7 @@ def numbuf_serialize(value): The serialized object. """ assert len(contained_objectids) == 0, "This should be unreachable." - return libnumbuf.serialize_list([value]) + return numbuf.serialize_list([value]) class RayTaskError(Exception): """An object used internally to represent a task that threw an exception. @@ -419,7 +419,7 @@ class Worker(object): # write the metadata metadata[:] = schema data = np.frombuffer(buff, dtype="byte")[8 + len(schema):] - metadata_offset = libnumbuf.write_to_buffer(serialized, memoryview(data)) + metadata_offset = numbuf.write_to_buffer(serialized, memoryview(data)) raylib.finish_buffer(self.handle, objectid, segmentid, metadata_offset) def get_object(self, objectid): @@ -436,7 +436,7 @@ class Worker(object): metadata_size = int(np.frombuffer(buff, dtype="int64", count=1)[0]) metadata = np.frombuffer(buff, dtype="byte", offset=8, count=metadata_size) data = np.frombuffer(buff, dtype="byte")[8 + metadata_size:] - serialized = libnumbuf.read_from_buffer(memoryview(data), bytearray(metadata), metadata_offset) + serialized = numbuf.read_from_buffer(memoryview(data), bytearray(metadata), metadata_offset) # If there is currently no ObjectFixture for this ObjectID, then create a # new one. The object_fixtures object is a WeakValueDictionary, so entries # will be discarded when there are no strong references to their values. @@ -446,7 +446,7 @@ class Worker(object): if objectid.id not in object_fixtures: object_fixture = ObjectFixture(objectid, segmentid, self.handle) object_fixtures[objectid.id] = object_fixture - deserialized = libnumbuf.deserialize_list(serialized, object_fixtures[objectid.id]) + deserialized = numbuf.deserialize_list(serialized, object_fixtures[objectid.id]) # Unwrap the object from the list (it was wrapped put_object) assert len(deserialized) == 1 result = deserialized[0] @@ -855,11 +855,11 @@ def register_class(cls, pickle=False, worker=global_worker): """Enable workers to serialize or deserialize objects of a particular class. This method runs the register_class function defined below on every worker, - which will enable libnumbuf to properly serialize and deserialize objects of - this class. + which will enable numbuf to properly serialize and deserialize objects of this + class. Args: - cls (type): The class that libnumbuf should serialize. + cls (type): The class that numbuf should serialize. pickle (bool): If False then objects of this class will be serialized by turning their __dict__ fields into a dictionary. If True, then objects of this class will be serialized using pickle. diff --git a/test/runtest.py b/test/runtest.py index 03b502143..a02405a02 100644 --- a/test/runtest.py +++ b/test/runtest.py @@ -5,7 +5,6 @@ import time import string import sys from collections import namedtuple -import libnumbuf import test_functions import ray.array.remote as ra