diff --git a/.travis.yml b/.travis.yml index 4a3edc383..aa2f8acc2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,4 +24,3 @@ script: - cd test - python runtest.py - python array_test.py - - python datasets_test.py diff --git a/doc/install-on-macosx.md b/doc/install-on-macosx.md index 87989396f..132e048be 100644 --- a/doc/install-on-macosx.md +++ b/doc/install-on-macosx.md @@ -16,10 +16,10 @@ install [Anaconda](https://www.continuum.io/downloads). ``` brew update -brew install git cmake automake autoconf libtool boost libjpeg graphviz +brew install git cmake automake autoconf libtool boost graphviz sudo easy_install pip sudo pip install ipython --user -sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0a2 boto3 botocore Pillow colorama graphviz --ignore-installed six +sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0a2 colorama graphviz --ignore-installed six ``` ### Build diff --git a/doc/install-on-ubuntu.md b/doc/install-on-ubuntu.md index ab4d11ab4..f0d2cd00d 100644 --- a/doc/install-on-ubuntu.md +++ b/doc/install-on-ubuntu.md @@ -14,8 +14,8 @@ First install the dependencies. We currently do not support Python 3. ``` sudo apt-get update -sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip libjpeg8-dev graphviz -sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0a2 boto3 botocore Pillow colorama graphviz +sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip graphviz +sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0a2 colorama graphviz ``` ### Build diff --git a/examples/imagenet/README.md b/examples/imagenet/README.md new file mode 100644 index 000000000..39bf9f11d --- /dev/null +++ b/examples/imagenet/README.md @@ -0,0 +1,14 @@ +Dependencies for Imagenet + +**On Ubuntu** + +``` +sudo apt-get install libjpeg8-dev +sudo pip install boto3 botocore pillow +``` +**On Mac OSX** + +``` +brew install libjpeg +sudo pip install boto3 botocore pillow +``` diff --git a/examples/imagenet/driver.py b/examples/imagenet/driver.py index 94bf99db7..5024f4c7c 100644 --- a/examples/imagenet/driver.py +++ b/examples/imagenet/driver.py @@ -3,7 +3,7 @@ import boto3 import os import numpy as np import ray -import ray.datasets.imagenet as imagenet +import imagenet import functions diff --git a/examples/imagenet/functions.py b/examples/imagenet/functions.py index 7f9ed89f0..e90f72a40 100644 --- a/examples/imagenet/functions.py +++ b/examples/imagenet/functions.py @@ -1,18 +1,18 @@ import numpy as np -from typing import List +from typing import List, Tuple import ray import ray.array.remote as ra -@ray.remote([List[ray.ObjRef]], [int]) +@ray.remote([List[Tuple[ray.ObjRef, ray.ObjRef]]], [int]) def num_images(batches): - shape_refs = [ra.shape(batch) for batch in batches] + shape_refs = [ra.shape(batch[0]) for batch in batches] return sum([ray.get(shape_ref)[0] for shape_ref in shape_refs]) -@ray.remote([List[ray.ObjRef]], [np.ndarray]) +@ray.remote([List[Tuple[ray.ObjRef, ray.ObjRef]]], [np.ndarray]) def compute_mean_image(batches): if len(batches) == 0: raise Exception("No images were passed into `compute_mean_image`.") - sum_image_refs = [ra.sum(batch, axis=0) for batch in batches] + sum_image_refs = [ra.sum(batch[0], axis=0) for batch in batches] sum_images = [ray.get(ref) for ref in sum_image_refs] n_images = num_images(batches) return np.sum(sum_images, axis=0).astype("float64") / ray.get(n_images) diff --git a/lib/python/ray/datasets/imagenet.py b/examples/imagenet/imagenet.py similarity index 89% rename from lib/python/ray/datasets/imagenet.py rename to examples/imagenet/imagenet.py index ea8456e5c..af9df9f71 100644 --- a/lib/python/ray/datasets/imagenet.py +++ b/examples/imagenet/imagenet.py @@ -1,5 +1,5 @@ import tarfile, io -from typing import List +from typing import List, Tuple import PIL.Image import numpy as np import boto3 @@ -21,6 +21,7 @@ def load_chunk(tarfile, size=None): """ result = [] + filenames = [] for member in tarfile.getmembers(): filename = member.path content = tarfile.extractfile(member) @@ -30,9 +31,10 @@ def load_chunk(tarfile, size=None): if size != None: rgbimg = rgbimg.resize(size, PIL.Image.ANTIALIAS) result.append(np.array(rgbimg).reshape(1, rgbimg.size[0], rgbimg.size[1], 3)) - return np.concatenate(result) + filenames.append(filename) + return np.concatenate(result), filenames -@ray.remote([str, str, List[int]], [np.ndarray]) +@ray.remote([str, str, List[int]], [np.ndarray, List]) def load_tarfile_from_s3(bucket, s3_key, size=[]): """Load an imagenet .tar file. @@ -55,7 +57,7 @@ def load_tarfile_from_s3(bucket, s3_key, size=[]): tar = tarfile.open(mode="r", fileobj=output) return load_chunk(tar, size=size if size != [] else None) -@ray.remote([str, List[str], List[int]], [List[ray.ObjRef]]) +@ray.remote([str, List[str], List[int]], [List[Tuple[ray.ObjRef, ray.ObjRef]]]) def load_tarfiles_from_s3(bucket, s3_keys, size=[]): """Load a number of imagenet .tar files. diff --git a/examples/imagenet/worker.py b/examples/imagenet/worker.py index 2203a1707..233dac2f1 100644 --- a/examples/imagenet/worker.py +++ b/examples/imagenet/worker.py @@ -2,7 +2,7 @@ import sys import argparse import numpy as np -import ray.datasets.imagenet +import imagenet import ray import ray.array.remote as ra @@ -19,7 +19,7 @@ if __name__ == "__main__": args = parser.parse_args() ray.worker.connect(args.scheduler_address, args.objstore_address, args.worker_address) - ray.register_module(ray.datasets.imagenet) + ray.register_module(imagenet) ray.register_module(functions) ray.register_module(ra) ray.register_module(ra.random) diff --git a/install-dependencies.sh b/install-dependencies.sh index 6da1807f6..63a69b285 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -30,12 +30,12 @@ fi if [[ $platform == "linux" ]]; then # These commands must be kept in sync with the installation instructions. sudo apt-get update - sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip libjpeg8-dev graphviz - sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0a2 boto3 botocore Pillow colorama graphviz + sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip graphviz + sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0a2 colorama graphviz elif [[ $platform == "macosx" ]]; then # These commands must be kept in sync with the installation instructions. - brew install git cmake automake autoconf libtool boost libjpeg graphviz + brew install git cmake automake autoconf libtool boost graphviz sudo easy_install pip sudo pip install ipython --user - sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0-alpha-2 boto3 botocore Pillow colorama graphviz --ignore-installed six + sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0-alpha-2 colorama graphviz --ignore-installed six fi diff --git a/lib/python/ray/datasets/__init__.py b/lib/python/ray/datasets/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/python/ray/worker.py b/lib/python/ray/worker.py index 3c1f5fcea..3d8c214be 100644 --- a/lib/python/ray/worker.py +++ b/lib/python/ray/worker.py @@ -233,7 +233,7 @@ def get(objref, worker=global_worker): print_task_info(ray.lib.task_info(worker.handle), worker.mode) value = worker.get_object(objref) if isinstance(value, RayFailedObject): - raise Exception("The task that created this object reference failed with error message: {}".format(value.error_message)) + raise Exception("The task that created this object reference failed with error message:\n{}".format(value.error_message)) return value def put(value, worker=global_worker): diff --git a/test/datasets_test.py b/test/datasets_test.py deleted file mode 100644 index bf991a970..000000000 --- a/test/datasets_test.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import tarfile -import unittest -import ray -import ray.datasets.imagenet as imagenet - -class ImageNetTest(unittest.TestCase): - - def testImageNetLoading(self): - worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_worker.py") - ray.services.start_ray_local(num_workers=5, worker_path=worker_path) - - chunk_name = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../data/mini.tar") - tar = tarfile.open(chunk_name, mode= "r") - chunk = imagenet.load_chunk(tar, size=(256, 256)) - self.assertEqual(chunk.shape, (2, 256, 256, 3)) - - ray.services.cleanup() - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_worker.py b/test/test_worker.py index dfdbe2945..45a80181e 100644 --- a/test/test_worker.py +++ b/test/test_worker.py @@ -5,7 +5,6 @@ import numpy as np import test_functions import ray.array.remote as ra import ray.array.distributed as da -import ray.datasets.imagenet import ray